From 4798ab19245f22622b16120523537e87c31fd273 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sun, 2 Dec 2018 01:25:33 +0800 Subject: [PATCH 01/13] Use cluster connections pool in DirectoryMonitor --- .../Storages/Distributed/DirectoryMonitor.cpp | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 74821f432e7..7cff6705fa5 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -175,17 +175,33 @@ void StorageDistributedDirectoryMonitor::run() ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::string & name, const StorageDistributed & storage) { auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.context.getSettingsRef()); - const auto pool_factory = [&storage, &timeouts] (const std::string & host, const UInt16 port, - const Protocol::Secure secure, - const std::string & user, const std::string & password, - const std::string & default_database) + const auto pool_factory = [&storage, &timeouts](const std::string & host, const UInt16 port, + const Protocol::Secure secure, + const std::string & user, const std::string & password, + const std::string & default_database) -> ConnectionPoolPtr { - return std::make_shared( - 1, host, port, default_database, - user, password, timeouts, - storage.getName() + '_' + user, - Protocol::Compression::Enable, - secure); + ClusterPtr cluster = storage.getCluster(); + const auto shards_info = cluster->getShardsInfo(); + const auto shards_addresses = cluster->getShardsAddresses(); + + /// existing connections pool have a higher priority + for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) + { + Cluster::Addresses replicas_addresses = shards_addresses[shard_index]; + + for (size_t replica_index = 0; replica_index < replicas_addresses.size(); ++replica_index) + { + Cluster::Address replica_address = replicas_addresses[replica_index]; + + if (replica_address.host_name == host && replica_address.port == port + && replica_address.secure == secure && replica_address.user == user + && replica_address.password == password && replica_address.default_database == default_database) + return shards_info[shard_index].per_replica_pools[replica_index]; + } + } + + return std::make_shared(1, host, port, default_database, user, password, timeouts, + storage.getName() + '_' + user, Protocol::Compression::Enable, secure); }; auto pools = createPoolsForAddresses(name, pool_factory); From 994d1c0fe04800dd8ab9bca0d8856c67b3b75fb6 Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Sun, 2 Dec 2018 10:17:08 +0800 Subject: [PATCH 02/13] fix excessive copy --- dbms/src/Interpreters/Cluster.cpp | 43 ++++++++++++- dbms/src/Interpreters/Cluster.h | 5 +- .../Storages/Distributed/DirectoryMonitor.cpp | 61 ++++--------------- .../Storages/System/StorageSystemClusters.cpp | 51 ++++++---------- 4 files changed, 78 insertions(+), 82 deletions(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 4bf446b107b..b562f986bee 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -122,6 +122,47 @@ String Cluster::Address::toStringFull() const + ((secure == Protocol::Secure::Enable) ? "+secure" : ""); } +void Cluster::Address::fromFullString(const String & full_string, Cluster::Address & address) +{ + const char * address_begin = full_string.data(); + const char * address_end = address_begin + full_string.size(); + + Protocol::Secure secure = Protocol::Secure::Disable; + const char * secure_tag = "+secure"; + if (endsWith(full_string, secure_tag)) + { + address_end -= strlen(secure_tag); + secure = Protocol::Secure::Enable; + } + + const char * user_pw_end = strchr(full_string.data(), '@'); + const char * colon = strchr(full_string.data(), ':'); + if (!user_pw_end || !colon) + throw Exception("Incorrect user[:password]@host:port#default_database format " + full_string, ErrorCodes::SYNTAX_ERROR); + + const bool has_pw = colon < user_pw_end; + const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon; + if (!host_end) + throw Exception("Incorrect address '" + full_string + "', it does not contain port", ErrorCodes::SYNTAX_ERROR); + + const char * has_db = strchr(full_string.data(), '#'); + const char * port_end = has_db ? has_db : address_end; + + address.secure = secure; + address.port = parse(host_end + 1, port_end - (host_end + 1)); + address.host_name = unescapeForFileName(std::string(user_pw_end + 1, host_end)); + address.user = unescapeForFileName(std::string(address_begin, has_pw ? colon : user_pw_end)); + address.password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string(); + address.default_database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end)) : std::string(); +} + +bool Cluster::Address::operator==(const Cluster::Address & other) const +{ + return other.host_name == host_name && other.port == port + && other.secure == secure && other.user == user + && other.password == password && other.default_database == default_database; +} + /// Implementation of Clusters class @@ -198,7 +239,6 @@ Cluster::Cluster(Poco::Util::AbstractConfiguration & config, const Settings & se const auto weight = config.getInt(prefix + ".weight", default_weight); addresses.emplace_back(config, prefix); - addresses.back().replica_num = 1; const auto & address = addresses.back(); ShardInfo info; @@ -253,7 +293,6 @@ Cluster::Cluster(Poco::Util::AbstractConfiguration & config, const Settings & se if (startsWith(replica_key, "replica")) { replica_addresses.emplace_back(config, partial_prefix + replica_key); - replica_addresses.back().replica_num = current_replica_num; ++current_replica_num; if (!replica_addresses.back().is_local) diff --git a/dbms/src/Interpreters/Cluster.h b/dbms/src/Interpreters/Cluster.h index 8bfbc073c61..4338447dbd7 100644 --- a/dbms/src/Interpreters/Cluster.h +++ b/dbms/src/Interpreters/Cluster.h @@ -59,7 +59,6 @@ public: String password; /// This database is selected when no database is specified for Distributed table String default_database; - UInt32 replica_num; /// The locality is determined at the initialization, and is not changed even if DNS is changed bool is_local; Protocol::Compression compression = Protocol::Compression::Enable; @@ -82,12 +81,16 @@ public: /// Retrurns escaped user:password@resolved_host_address:resolved_host_port#default_database String toStringFull() const; + static void fromFullString(const String & address_full_string, Address & address); + /// Returns initially resolved address Poco::Net::SocketAddress getResolvedAddress() const { return initially_resolved_address; } + bool operator==(const Address & other) const; + private: Poco::Net::SocketAddress initially_resolved_address; }; diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 7cff6705fa5..9ef4e012d8f 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -48,40 +48,9 @@ namespace for (auto it = boost::make_split_iterator(name, boost::first_finder(",")); it != decltype(it){}; ++it) { - const auto address = boost::copy_range(*it); - const char * address_begin = static_cast(address.data()); - const char * address_end = address_begin + address.size(); - - Protocol::Secure secure = Protocol::Secure::Disable; - const char * secure_tag = "+secure"; - if (endsWith(address, secure_tag)) - { - address_end -= strlen(secure_tag); - secure = Protocol::Secure::Enable; - } - - const char * user_pw_end = strchr(address.data(), '@'); - const char * colon = strchr(address.data(), ':'); - if (!user_pw_end || !colon) - throw Exception{"Shard address '" + address + "' does not match to 'user[:password]@host:port#default_database' pattern", - ErrorCodes::INCORRECT_FILE_NAME}; - - const bool has_pw = colon < user_pw_end; - const char * host_end = has_pw ? strchr(user_pw_end + 1, ':') : colon; - if (!host_end) - throw Exception{"Shard address '" + address + "' does not contain port", ErrorCodes::INCORRECT_FILE_NAME}; - - const char * has_db = strchr(address.data(), '#'); - const char * port_end = has_db ? has_db : address_end; - - const auto user = unescapeForFileName(std::string(address_begin, has_pw ? colon : user_pw_end)); - const auto password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string(); - const auto host = unescapeForFileName(std::string(user_pw_end + 1, host_end)); - const auto port = parse(host_end + 1, port_end - (host_end + 1)); - const auto database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end)) - : std::string(); - - pools.emplace_back(factory(host, port, secure, user, password, database)); + Cluster::Address address; + Cluster::Address::fromFullString(boost::copy_range(*it), address); + pools.emplace_back(factory(address)); } return pools; @@ -175,33 +144,29 @@ void StorageDistributedDirectoryMonitor::run() ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::string & name, const StorageDistributed & storage) { auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.context.getSettingsRef()); - const auto pool_factory = [&storage, &timeouts](const std::string & host, const UInt16 port, - const Protocol::Secure secure, - const std::string & user, const std::string & password, - const std::string & default_database) -> ConnectionPoolPtr + const auto pool_factory = [&storage, &timeouts](const Cluster::Address & address) -> ConnectionPoolPtr { - ClusterPtr cluster = storage.getCluster(); - const auto shards_info = cluster->getShardsInfo(); - const auto shards_addresses = cluster->getShardsAddresses(); + const auto & cluster = storage.getCluster(); + const auto & shards_info = cluster->getShardsInfo(); + const auto & shards_addresses = cluster->getShardsAddresses(); /// existing connections pool have a higher priority for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) { - Cluster::Addresses replicas_addresses = shards_addresses[shard_index]; + const Cluster::Addresses & replicas_addresses = shards_addresses[shard_index]; for (size_t replica_index = 0; replica_index < replicas_addresses.size(); ++replica_index) { - Cluster::Address replica_address = replicas_addresses[replica_index]; + const Cluster::Address & replica_address = replicas_addresses[replica_index]; - if (replica_address.host_name == host && replica_address.port == port - && replica_address.secure == secure && replica_address.user == user - && replica_address.password == password && replica_address.default_database == default_database) + if (address == replica_address) return shards_info[shard_index].per_replica_pools[replica_index]; } } - return std::make_shared(1, host, port, default_database, user, password, timeouts, - storage.getName() + '_' + user, Protocol::Compression::Enable, secure); + return std::make_shared( + 1, address.host_name, address.port, address.default_database, address.user, address.password, timeouts, + storage.getName() + '_' + address.user, Protocol::Compression::Enable, address.secure); }; auto pools = createPoolsForAddresses(name, pool_factory); diff --git a/dbms/src/Storages/System/StorageSystemClusters.cpp b/dbms/src/Storages/System/StorageSystemClusters.cpp index b0ad56e8eb5..b33b2d86d0e 100644 --- a/dbms/src/Storages/System/StorageSystemClusters.cpp +++ b/dbms/src/Storages/System/StorageSystemClusters.cpp @@ -26,44 +26,33 @@ NamesAndTypesList StorageSystemClusters::getNamesAndTypes() void StorageSystemClusters::fillData(MutableColumns & res_columns, const Context & context, const SelectQueryInfo &) const { - auto updateColumns = [&](const std::string & cluster_name, const Cluster::ShardInfo & shard_info, const Cluster::Address & address) + for (const auto & name_and_cluster : context.getClusters().getContainer()) { - size_t i = 0; - res_columns[i++]->insert(cluster_name); - res_columns[i++]->insert(shard_info.shard_num); - res_columns[i++]->insert(shard_info.weight); - res_columns[i++]->insert(address.replica_num); - res_columns[i++]->insert(address.host_name); - res_columns[i++]->insert(DNSResolver::instance().resolveHost(address.host_name).toString()); - res_columns[i++]->insert(address.port); - res_columns[i++]->insert(shard_info.isLocal()); - res_columns[i++]->insert(address.user); - res_columns[i++]->insert(address.default_database); - }; - - auto clusters = context.getClusters().getContainer(); - for (const auto & entry : clusters) - { - const std::string cluster_name = entry.first; - const ClusterPtr cluster = entry.second; - const auto & addresses_with_failover = cluster->getShardsAddresses(); + const String & cluster_name = name_and_cluster.first; + const ClusterPtr & cluster = name_and_cluster.second; const auto & shards_info = cluster->getShardsInfo(); + const auto & addresses_with_failover = cluster->getShardsAddresses(); - if (!addresses_with_failover.empty()) + for (size_t shard_index = 0; shard_index < shards_info.size(); ++shard_index) { - auto it1 = addresses_with_failover.cbegin(); - auto it2 = shards_info.cbegin(); + const auto & shard_info = shards_info[shard_index]; + const auto & shard_addresses = addresses_with_failover[shard_index]; - while (it1 != addresses_with_failover.cend()) + for (size_t replica_index = 0; replica_index < shard_addresses.size(); ++replica_index) { - const auto & addresses = *it1; - const auto & shard_info = *it2; + size_t i = 0; + const auto & address = shard_addresses[replica_index]; - for (const auto & address : addresses) - updateColumns(cluster_name, shard_info, address); - - ++it1; - ++it2; + res_columns[i++]->insert(cluster_name); + res_columns[i++]->insert(shard_info.shard_num); + res_columns[i++]->insert(shard_info.weight); + res_columns[i++]->insert(replica_index + 1); + res_columns[i++]->insert(address.host_name); + res_columns[i++]->insert(DNSResolver::instance().resolveHost(address.host_name).toString()); + res_columns[i++]->insert(address.port); + res_columns[i++]->insert(shard_info.isLocal()); + res_columns[i++]->insert(address.user); + res_columns[i++]->insert(address.default_database); } } } From 998591b30d7ade8d394b3d17dae358ab72e1d6ec Mon Sep 17 00:00:00 2001 From: zhang2014 Date: Fri, 18 Jan 2019 10:04:16 +0800 Subject: [PATCH 03/13] fix build failure --- dbms/src/Storages/Distributed/DirectoryMonitor.cpp | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 1c812909676..29a77557c6b 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -144,10 +144,7 @@ void StorageDistributedDirectoryMonitor::run() ConnectionPoolPtr StorageDistributedDirectoryMonitor::createPool(const std::string & name, const StorageDistributed & storage) { auto timeouts = ConnectionTimeouts::getTCPTimeoutsWithFailover(storage.global_context.getSettingsRef()); - const auto pool_factory = [&storage, &timeouts] (const std::string & host, const UInt16 port, - const Protocol::Secure secure, - const std::string & user, const std::string & password, - const std::string & default_database) + const auto pool_factory = [&storage, &timeouts] (const Cluster::Address & address) -> ConnectionPoolPtr { const auto & cluster = storage.getCluster(); const auto & shards_info = cluster->getShardsInfo(); From c7b95b5175b9b47451ca1bd02af2b93034df0caf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 20 Jan 2019 02:27:52 +0300 Subject: [PATCH 04/13] Miscellaneous changes for PVS-Studio --- dbms/programs/client/ConnectionParameters.h | 6 +++--- dbms/programs/client/TestHint.h | 7 +++---- dbms/src/AggregateFunctions/AggregateFunctionRetention.h | 2 +- dbms/src/Dictionaries/CacheDictionary.cpp | 5 ----- dbms/src/Dictionaries/CacheDictionary.h | 7 ++++--- dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp | 4 ---- dbms/src/Dictionaries/ComplexKeyCacheDictionary.h | 7 ++++--- dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp | 6 ------ dbms/src/Dictionaries/ComplexKeyHashedDictionary.h | 7 ++++--- dbms/src/Dictionaries/FlatDictionary.cpp | 6 ------ dbms/src/Dictionaries/FlatDictionary.h | 7 ++++--- dbms/src/Dictionaries/HashedDictionary.cpp | 6 ------ dbms/src/Dictionaries/HashedDictionary.h | 7 ++++--- dbms/src/Dictionaries/IDictionarySource.h | 1 + dbms/src/Dictionaries/RangeHashedDictionary.cpp | 6 ------ dbms/src/Dictionaries/RangeHashedDictionary.h | 7 ++++--- dbms/src/Dictionaries/TrieDictionary.cpp | 5 ----- dbms/src/Dictionaries/TrieDictionary.h | 7 ++++--- dbms/src/Functions/if.cpp | 2 +- dbms/src/Interpreters/IExternalLoadable.h | 3 ++- dbms/src/Parsers/parseQuery.cpp | 2 -- 21 files changed, 39 insertions(+), 71 deletions(-) diff --git a/dbms/programs/client/ConnectionParameters.h b/dbms/programs/client/ConnectionParameters.h index 68bc3728349..557929a9331 100644 --- a/dbms/programs/client/ConnectionParameters.h +++ b/dbms/programs/client/ConnectionParameters.h @@ -25,12 +25,12 @@ namespace ErrorCodes struct ConnectionParameters { String host; - UInt16 port; + UInt16 port{}; String default_database; String user; String password; - Protocol::Secure security; - Protocol::Compression compression; + Protocol::Secure security = Protocol::Secure::Disable; + Protocol::Compression compression = Protocol::Compression::Enable; ConnectionTimeouts timeouts; ConnectionParameters() {} diff --git a/dbms/programs/client/TestHint.h b/dbms/programs/client/TestHint.h index c1ac913e7a8..66e86b5e750 100644 --- a/dbms/programs/client/TestHint.h +++ b/dbms/programs/client/TestHint.h @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include @@ -91,14 +93,11 @@ private: void parse(const String & hint) { - std::stringstream ss; - ss << hint; + ReadBufferFromString ss(hint); while (!ss.eof()) { String item; ss >> item; - if (item.empty()) - break; if (item == "serverError") ss >> server_error; diff --git a/dbms/src/AggregateFunctions/AggregateFunctionRetention.h b/dbms/src/AggregateFunctions/AggregateFunctionRetention.h index 8fac497bdfe..688f7f1404c 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionRetention.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionRetention.h @@ -86,7 +86,7 @@ public: ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT}; } - events_size = arguments.size(); + events_size = static_cast(arguments.size()); } diff --git a/dbms/src/Dictionaries/CacheDictionary.cpp b/dbms/src/Dictionaries/CacheDictionary.cpp index cd61d862c20..752752fae59 100644 --- a/dbms/src/Dictionaries/CacheDictionary.cpp +++ b/dbms/src/Dictionaries/CacheDictionary.cpp @@ -81,11 +81,6 @@ CacheDictionary::CacheDictionary( createAttributes(); } -CacheDictionary::CacheDictionary(const CacheDictionary & other) - : CacheDictionary{other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime, other.size} -{ -} - void CacheDictionary::toParent(const PaddedPODArray & ids, PaddedPODArray & out) const { diff --git a/dbms/src/Dictionaries/CacheDictionary.h b/dbms/src/Dictionaries/CacheDictionary.h index 9122b0d82bc..7a08ad5c3ab 100644 --- a/dbms/src/Dictionaries/CacheDictionary.h +++ b/dbms/src/Dictionaries/CacheDictionary.h @@ -30,8 +30,6 @@ public: const DictionaryLifetime dict_lifetime, const size_t size); - CacheDictionary(const CacheDictionary & other); - std::exception_ptr getCreationException() const override { return {}; } std::string getName() const override { return name; } @@ -53,7 +51,10 @@ public: bool isCached() const override { return true; } - std::unique_ptr clone() const override { return std::make_unique(*this); } + std::unique_ptr clone() const override + { + return std::make_unique(name, dict_struct, source_ptr->clone(), dict_lifetime, size); + } const IDictionarySource * getSource() const override { return source_ptr.get(); } diff --git a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp index 84ab8c569a4..42aa0c943c7 100644 --- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp +++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.cpp @@ -70,10 +70,6 @@ ComplexKeyCacheDictionary::ComplexKeyCacheDictionary( createAttributes(); } -ComplexKeyCacheDictionary::ComplexKeyCacheDictionary(const ComplexKeyCacheDictionary & other) - : ComplexKeyCacheDictionary{other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime, other.size} -{ -} void ComplexKeyCacheDictionary::getString( const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ColumnString * out) const diff --git a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h index 017a638d776..92666158015 100644 --- a/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h +++ b/dbms/src/Dictionaries/ComplexKeyCacheDictionary.h @@ -47,8 +47,6 @@ public: const DictionaryLifetime dict_lifetime, const size_t size); - ComplexKeyCacheDictionary(const ComplexKeyCacheDictionary & other); - std::string getKeyDescription() const { return key_description; } std::exception_ptr getCreationException() const override { return {}; } @@ -76,7 +74,10 @@ public: bool isCached() const override { return true; } - std::unique_ptr clone() const override { return std::make_unique(*this); } + std::unique_ptr clone() const override + { + return std::make_unique(name, dict_struct, source_ptr->clone(), dict_lifetime, size); + } const IDictionarySource * getSource() const override { return source_ptr.get(); } diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp index 8422fb2a68f..a36b225680d 100644 --- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp +++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.cpp @@ -43,12 +43,6 @@ ComplexKeyHashedDictionary::ComplexKeyHashedDictionary( creation_time = std::chrono::system_clock::now(); } -ComplexKeyHashedDictionary::ComplexKeyHashedDictionary(const ComplexKeyHashedDictionary & other) - : ComplexKeyHashedDictionary{ - other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime, other.require_nonempty, other.saved_block} -{ -} - #define DECLARE(TYPE) \ void ComplexKeyHashedDictionary::get##TYPE( \ const std::string & attribute_name, const Columns & key_columns, const DataTypes & key_types, ResultArrayType & out) const \ diff --git a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h index 8362d1f6f4a..fc1fc709388 100644 --- a/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h +++ b/dbms/src/Dictionaries/ComplexKeyHashedDictionary.h @@ -29,8 +29,6 @@ public: bool require_nonempty, BlockPtr saved_block = nullptr); - ComplexKeyHashedDictionary(const ComplexKeyHashedDictionary & other); - std::string getKeyDescription() const { return key_description; } std::exception_ptr getCreationException() const override { return creation_exception; } @@ -51,7 +49,10 @@ public: bool isCached() const override { return false; } - std::unique_ptr clone() const override { return std::make_unique(*this); } + std::unique_ptr clone() const override + { + return std::make_unique(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block); + } const IDictionarySource * getSource() const override { return source_ptr.get(); } diff --git a/dbms/src/Dictionaries/FlatDictionary.cpp b/dbms/src/Dictionaries/FlatDictionary.cpp index 32fa5085f63..26640972014 100644 --- a/dbms/src/Dictionaries/FlatDictionary.cpp +++ b/dbms/src/Dictionaries/FlatDictionary.cpp @@ -50,12 +50,6 @@ FlatDictionary::FlatDictionary( creation_time = std::chrono::system_clock::now(); } -FlatDictionary::FlatDictionary(const FlatDictionary & other) - : FlatDictionary{ - other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime, other.require_nonempty, other.saved_block} -{ -} - void FlatDictionary::toParent(const PaddedPODArray & ids, PaddedPODArray & out) const { diff --git a/dbms/src/Dictionaries/FlatDictionary.h b/dbms/src/Dictionaries/FlatDictionary.h index a5f1fe4b4fc..ccdf522162e 100644 --- a/dbms/src/Dictionaries/FlatDictionary.h +++ b/dbms/src/Dictionaries/FlatDictionary.h @@ -28,8 +28,6 @@ public: bool require_nonempty, BlockPtr saved_block = nullptr); - FlatDictionary(const FlatDictionary & other); - std::exception_ptr getCreationException() const override { return creation_exception; } std::string getName() const override { return name; } @@ -48,7 +46,10 @@ public: bool isCached() const override { return false; } - std::unique_ptr clone() const override { return std::make_unique(*this); } + std::unique_ptr clone() const override + { + return std::make_unique(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block); + } const IDictionarySource * getSource() const override { return source_ptr.get(); } diff --git a/dbms/src/Dictionaries/HashedDictionary.cpp b/dbms/src/Dictionaries/HashedDictionary.cpp index 47817af25f5..6154b9243ff 100644 --- a/dbms/src/Dictionaries/HashedDictionary.cpp +++ b/dbms/src/Dictionaries/HashedDictionary.cpp @@ -44,12 +44,6 @@ HashedDictionary::HashedDictionary( creation_time = std::chrono::system_clock::now(); } -HashedDictionary::HashedDictionary(const HashedDictionary & other) - : HashedDictionary{ - other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime, other.require_nonempty, other.saved_block} -{ -} - void HashedDictionary::toParent(const PaddedPODArray & ids, PaddedPODArray & out) const { diff --git a/dbms/src/Dictionaries/HashedDictionary.h b/dbms/src/Dictionaries/HashedDictionary.h index 053af0eac0c..0ced0726b99 100644 --- a/dbms/src/Dictionaries/HashedDictionary.h +++ b/dbms/src/Dictionaries/HashedDictionary.h @@ -27,8 +27,6 @@ public: bool require_nonempty, BlockPtr saved_block = nullptr); - HashedDictionary(const HashedDictionary & other); - std::exception_ptr getCreationException() const override { return creation_exception; } std::string getName() const override { return name; } @@ -47,7 +45,10 @@ public: bool isCached() const override { return false; } - std::unique_ptr clone() const override { return std::make_unique(*this); } + std::unique_ptr clone() const override + { + return std::make_unique(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty, saved_block); + } const IDictionarySource * getSource() const override { return source_ptr.get(); } diff --git a/dbms/src/Dictionaries/IDictionarySource.h b/dbms/src/Dictionaries/IDictionarySource.h index 23f0a436e19..bc2df7c3c56 100644 --- a/dbms/src/Dictionaries/IDictionarySource.h +++ b/dbms/src/Dictionaries/IDictionarySource.h @@ -3,6 +3,7 @@ #include #include + namespace DB { class IDictionarySource; diff --git a/dbms/src/Dictionaries/RangeHashedDictionary.cpp b/dbms/src/Dictionaries/RangeHashedDictionary.cpp index 1ee3b806287..48c884fa773 100644 --- a/dbms/src/Dictionaries/RangeHashedDictionary.cpp +++ b/dbms/src/Dictionaries/RangeHashedDictionary.cpp @@ -94,12 +94,6 @@ RangeHashedDictionary::RangeHashedDictionary( creation_time = std::chrono::system_clock::now(); } -RangeHashedDictionary::RangeHashedDictionary(const RangeHashedDictionary & other) - : RangeHashedDictionary{ - other.dictionary_name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime, other.require_nonempty} -{ -} - #define DECLARE_MULTIPLE_GETTER(TYPE) \ void RangeHashedDictionary::get##TYPE( \ diff --git a/dbms/src/Dictionaries/RangeHashedDictionary.h b/dbms/src/Dictionaries/RangeHashedDictionary.h index a5043d40d5d..53e48ffe7eb 100644 --- a/dbms/src/Dictionaries/RangeHashedDictionary.h +++ b/dbms/src/Dictionaries/RangeHashedDictionary.h @@ -24,8 +24,6 @@ public: const DictionaryLifetime dict_lifetime, bool require_nonempty); - RangeHashedDictionary(const RangeHashedDictionary & other); - std::exception_ptr getCreationException() const override { return creation_exception; } std::string getName() const override { return dictionary_name; } @@ -44,7 +42,10 @@ public: bool isCached() const override { return false; } - std::unique_ptr clone() const override { return std::make_unique(*this); } + std::unique_ptr clone() const override + { + return std::make_unique(dictionary_name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty); + } const IDictionarySource * getSource() const override { return source_ptr.get(); } diff --git a/dbms/src/Dictionaries/TrieDictionary.cpp b/dbms/src/Dictionaries/TrieDictionary.cpp index 00bafdf62e3..9dc5fc0891d 100644 --- a/dbms/src/Dictionaries/TrieDictionary.cpp +++ b/dbms/src/Dictionaries/TrieDictionary.cpp @@ -63,11 +63,6 @@ TrieDictionary::TrieDictionary( creation_time = std::chrono::system_clock::now(); } -TrieDictionary::TrieDictionary(const TrieDictionary & other) - : TrieDictionary{other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime, other.require_nonempty} -{ -} - TrieDictionary::~TrieDictionary() { btrie_destroy(trie); diff --git a/dbms/src/Dictionaries/TrieDictionary.h b/dbms/src/Dictionaries/TrieDictionary.h index 1d370c8080c..3191916c094 100644 --- a/dbms/src/Dictionaries/TrieDictionary.h +++ b/dbms/src/Dictionaries/TrieDictionary.h @@ -29,8 +29,6 @@ public: const DictionaryLifetime dict_lifetime, bool require_nonempty); - TrieDictionary(const TrieDictionary & other); - ~TrieDictionary() override; std::string getKeyDescription() const { return key_description; } @@ -53,7 +51,10 @@ public: bool isCached() const override { return false; } - std::unique_ptr clone() const override { return std::make_unique(*this); } + std::unique_ptr clone() const override + { + return std::make_unique(name, dict_struct, source_ptr->clone(), dict_lifetime, require_nonempty); + } const IDictionarySource * getSource() const override { return source_ptr.get(); } diff --git a/dbms/src/Functions/if.cpp b/dbms/src/Functions/if.cpp index 41a5277401c..64fe301291c 100644 --- a/dbms/src/Functions/if.cpp +++ b/dbms/src/Functions/if.cpp @@ -904,7 +904,7 @@ public: using T0 = typename Types::LeftType; using T1 = typename Types::RightType; - if constexpr ((IsDecimalNumber && IsDecimalNumber) || (!IsDecimalNumber && !IsDecimalNumber)) + if constexpr (IsDecimalNumber == IsDecimalNumber) return executeTyped(cond_col, block, arguments, result, input_rows_count); else throw Exception("Conditional function with Decimal and non Decimal", ErrorCodes::NOT_IMPLEMENTED); diff --git a/dbms/src/Interpreters/IExternalLoadable.h b/dbms/src/Interpreters/IExternalLoadable.h index 2353ac288e2..c94d8d97a49 100644 --- a/dbms/src/Interpreters/IExternalLoadable.h +++ b/dbms/src/Interpreters/IExternalLoadable.h @@ -3,6 +3,7 @@ #include #include #include +#include #include @@ -26,7 +27,7 @@ struct ExternalLoadableLifetime final /// Basic interface for external loadable objects. Is used in ExternalLoader. -class IExternalLoadable : public std::enable_shared_from_this +class IExternalLoadable : public std::enable_shared_from_this, private boost::noncopyable { public: virtual ~IExternalLoadable() = default; diff --git a/dbms/src/Parsers/parseQuery.cpp b/dbms/src/Parsers/parseQuery.cpp index f40d156f17d..4f8ab83b7fd 100644 --- a/dbms/src/Parsers/parseQuery.cpp +++ b/dbms/src/Parsers/parseQuery.cpp @@ -354,8 +354,6 @@ std::pair splitMultipartQuery(const std::string & queries, s begin = pos; ast = parseQueryAndMovePosition(parser, pos, end, "", true, 0); - if (!ast) - break; ASTInsertQuery * insert = typeid_cast(ast.get()); From a436915ed68afe7df979e0b3c10b631979d276b6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 20 Jan 2019 03:10:27 +0300 Subject: [PATCH 05/13] Fixed test #3960 --- .../00814_replicated_minimalistic_part_header_zookeeper.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sql b/dbms/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sql index 53905a72ab4..bd45ede270f 100644 --- a/dbms/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sql +++ b/dbms/tests/queries/0_stateless/00814_replicated_minimalistic_part_header_zookeeper.sql @@ -36,10 +36,10 @@ SELECT sleep(2) FORMAT Null; SELECT '*** Test part removal ***'; SELECT '*** replica 1 ***'; -SELECT name FROM system.parts WHERE database = 'test' AND table = 'part_header_r1'; +SELECT name FROM system.parts WHERE active AND database = 'test' AND table = 'part_header_r1'; SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/test/part_header/replicas/1/parts'; SELECT '*** replica 2 ***'; -SELECT name FROM system.parts WHERE database = 'test' AND table = 'part_header_r2'; +SELECT name FROM system.parts WHERE active AND database = 'test' AND table = 'part_header_r2'; SELECT name FROM system.zookeeper WHERE path = '/clickhouse/tables/test/part_header/replicas/1/parts'; SELECT '*** Test ALTER ***'; From 34fb1c89f1c4c5e57e249657fedd4359e45aa9b6 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jan 2019 17:00:06 +0300 Subject: [PATCH 06/13] Add ability to choose codecs for storage log and tiny log --- dbms/src/Storages/ColumnsDescription.cpp | 6 + dbms/src/Storages/ColumnsDescription.h | 2 + dbms/src/Storages/StorageLog.cpp | 11 +- dbms/src/Storages/StorageTinyLog.cpp | 7 +- ...m_compression_codes_log_storages.reference | 26 ++++ ..._custom_compression_codes_log_storages.sql | 118 ++++++++++++++++++ 6 files changed, 165 insertions(+), 5 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference create mode 100644 dbms/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index c64f78b330d..783d2d3397d 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -240,6 +240,12 @@ CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_ return codec->second; } + +CompressionCodecPtr ColumnsDescription::getCodecOrDefault(const String & column_name) const +{ + return getCodecOrDefault(column_name, CompressionCodecFactory::instance().getDefaultCodec()); +} + ColumnsDescription ColumnsDescription::parse(const String & str) { ReadBufferFromString buf{str}; diff --git a/dbms/src/Storages/ColumnsDescription.h b/dbms/src/Storages/ColumnsDescription.h index edeabf63dfb..e54dddde1de 100644 --- a/dbms/src/Storages/ColumnsDescription.h +++ b/dbms/src/Storages/ColumnsDescription.h @@ -69,6 +69,8 @@ struct ColumnsDescription CompressionCodecPtr getCodecOrDefault(const String & column_name, CompressionCodecPtr default_codec) const; + CompressionCodecPtr getCodecOrDefault(const String & column_name) const; + static ColumnsDescription parse(const String & str); static const ColumnsDescription * loadFromContext(const Context & context, const String & db, const String & table); diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index e04d02e7de9..5ee8726dd52 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -144,9 +144,9 @@ private: struct Stream { - Stream(const std::string & data_path, size_t max_compress_block_size) : + Stream(const std::string & data_path, CompressionCodecPtr codec, size_t max_compress_block_size) : plain(data_path, max_compress_block_size, O_APPEND | O_CREAT | O_WRONLY), - compressed(plain, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size) + compressed(plain, codec, max_compress_block_size) { plain_offset = Poco::File(data_path).getSize(); } @@ -355,7 +355,12 @@ void LogBlockOutputStream::writeData(const String & name, const IDataType & type if (written_streams.count(stream_name)) return; - streams.try_emplace(stream_name, storage.files[stream_name].data_file.path(), storage.max_compress_block_size); + const auto & columns = storage.getColumns(); + streams.try_emplace( + stream_name, + storage.files[stream_name].data_file.path(), + columns.getCodecOrDefault(name), + storage.max_compress_block_size); }, settings.path); settings.getter = createStreamGetter(name, written_streams); diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index d3259ff859c..9f87fb172fc 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -135,9 +135,9 @@ private: struct Stream { - Stream(const std::string & data_path, size_t max_compress_block_size) : + Stream(const std::string & data_path, CompressionCodecPtr codec, size_t max_compress_block_size) : plain(data_path, max_compress_block_size, O_APPEND | O_CREAT | O_WRONLY), - compressed(plain, CompressionCodecFactory::instance().getDefaultCodec(), max_compress_block_size) + compressed(plain, codec, max_compress_block_size) { } @@ -237,6 +237,7 @@ void TinyLogBlockInputStream::readData(const String & name, const IDataType & ty IDataType::OutputStreamGetter TinyLogBlockOutputStream::createStreamGetter(const String & name, WrittenStreams & written_streams) { + return [&] (const IDataType::SubstreamPath & path) -> WriteBuffer * { String stream_name = IDataType::getFileNameForStream(name, path); @@ -244,8 +245,10 @@ IDataType::OutputStreamGetter TinyLogBlockOutputStream::createStreamGetter(const if (!written_streams.insert(stream_name).second) return nullptr; + const auto & columns = storage.getColumns(); if (!streams.count(stream_name)) streams[stream_name] = std::make_unique(storage.files[stream_name].data_file.path(), + columns.getCodecOrDefault(name), storage.max_compress_block_size); return &streams[stream_name]->compressed; diff --git a/dbms/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference b/dbms/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference new file mode 100644 index 00000000000..8104bd3831f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.reference @@ -0,0 +1,26 @@ +CREATE TABLE test.compression_codec_log ( id UInt64 CODEC(LZ4), data String CODEC(ZSTD(1)), ddd Date CODEC(NONE), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), othernum Int64 CODEC(Delta(8))) ENGINE = Log() +1 hello 2018-12-14 1.1 aaa 5 +2 world 2018-12-15 2.2 bbb 6 +3 ! 2018-12-16 3.3 ccc 7 +2 +CREATE TABLE test.compression_codec_multiple_log ( id UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)), data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)), ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)), somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))) ENGINE = Log() +1 world 2018-10-05 1.1 +2 hello 2018-10-01 2.2 +3 buy 2018-10-11 3.3 +10003 +10003 +274972506.6 +9175437371954010821 +CREATE TABLE test.compression_codec_tiny_log ( id UInt64 CODEC(LZ4), data String CODEC(ZSTD(1)), ddd Date CODEC(NONE), somenum Float64 CODEC(ZSTD(2)), somestr FixedString(3) CODEC(LZ4HC(7)), othernum Int64 CODEC(Delta(8))) ENGINE = TinyLog() +1 hello 2018-12-14 1.1 aaa 5 +2 world 2018-12-15 2.2 bbb 6 +3 ! 2018-12-16 3.3 ccc 7 +2 +CREATE TABLE test.compression_codec_multiple_tiny_log ( id UInt64 CODEC(LZ4, ZSTD(1), NONE, LZ4HC(0), Delta(4)), data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC(0), LZ4, LZ4, Delta(8)), ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD(1), LZ4HC(0), LZ4HC(0)), somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD(1))) ENGINE = TinyLog() +1 world 2018-10-05 1.1 +2 hello 2018-10-01 2.2 +3 buy 2018-10-11 3.3 +10003 +10003 +274972506.6 +9175437371954010821 diff --git a/dbms/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql b/dbms/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql new file mode 100644 index 00000000000..7798cf7223e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00804_test_custom_compression_codes_log_storages.sql @@ -0,0 +1,118 @@ +SET send_logs_level = 'none'; + +-- copy-paste for storage log + +DROP TABLE IF EXISTS test.compression_codec_log; + +CREATE TABLE test.compression_codec_log( + id UInt64 CODEC(LZ4), + data String CODEC(ZSTD), + ddd Date CODEC(NONE), + somenum Float64 CODEC(ZSTD(2)), + somestr FixedString(3) CODEC(LZ4HC(7)), + othernum Int64 CODEC(Delta) +) ENGINE = Log(); + +SHOW CREATE TABLE test.compression_codec_log; + +INSERT INTO test.compression_codec_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); +INSERT INTO test.compression_codec_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); +INSERT INTO test.compression_codec_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); + +SELECT * FROM test.compression_codec_log ORDER BY id; + +INSERT INTO test.compression_codec_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); + +DETACH TABLE test.compression_codec_log; +ATTACH TABLE test.compression_codec_log; + +SELECT count(*) FROM test.compression_codec_log WHERE id = 2 GROUP BY id; + +DROP TABLE IF EXISTS test.compression_codec_log; + +DROP TABLE IF EXISTS test.compression_codec_multiple_log; + +CREATE TABLE test.compression_codec_multiple_log ( + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), + data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), + ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), + somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) +) ENGINE = Log(); + +SHOW CREATE TABLE test.compression_codec_multiple_log; + +INSERT INTO test.compression_codec_multiple_log VALUES (1, 'world', toDate('2018-10-05'), 1.1), (2, 'hello', toDate('2018-10-01'), 2.2), (3, 'buy', toDate('2018-10-11'), 3.3); + +SELECT * FROM test.compression_codec_multiple_log ORDER BY id; + +INSERT INTO test.compression_codec_multiple_log select modulo(number, 100), toString(number), toDate('2018-12-01'), 5.5 * number FROM system.numbers limit 10000; + +SELECT count(*) FROM test.compression_codec_multiple_log; + +SELECT count(distinct data) FROM test.compression_codec_multiple_log; + +SELECT floor(sum(somenum), 1) FROM test.compression_codec_multiple_log; + +TRUNCATE TABLE test.compression_codec_multiple_log; + +INSERT INTO test.compression_codec_multiple_log select modulo(number, 100), toString(number), toDate('2018-12-01'), 5.5 * number FROM system.numbers limit 10000; + +SELECT sum(cityHash64(*)) FROM test.compression_codec_multiple_log; + +-- copy-paste for storage tiny log +DROP TABLE IF EXISTS test.compression_codec_tiny_log; + +CREATE TABLE test.compression_codec_tiny_log( + id UInt64 CODEC(LZ4), + data String CODEC(ZSTD), + ddd Date CODEC(NONE), + somenum Float64 CODEC(ZSTD(2)), + somestr FixedString(3) CODEC(LZ4HC(7)), + othernum Int64 CODEC(Delta) +) ENGINE = TinyLog(); + +SHOW CREATE TABLE test.compression_codec_tiny_log; + +INSERT INTO test.compression_codec_tiny_log VALUES(1, 'hello', toDate('2018-12-14'), 1.1, 'aaa', 5); +INSERT INTO test.compression_codec_tiny_log VALUES(2, 'world', toDate('2018-12-15'), 2.2, 'bbb', 6); +INSERT INTO test.compression_codec_tiny_log VALUES(3, '!', toDate('2018-12-16'), 3.3, 'ccc', 7); + +SELECT * FROM test.compression_codec_tiny_log ORDER BY id; + +INSERT INTO test.compression_codec_tiny_log VALUES(2, '', toDate('2018-12-13'), 4.4, 'ddd', 8); + +DETACH TABLE test.compression_codec_tiny_log; +ATTACH TABLE test.compression_codec_tiny_log; + +SELECT count(*) FROM test.compression_codec_tiny_log WHERE id = 2 GROUP BY id; + +DROP TABLE IF EXISTS test.compression_codec_tiny_log; + +DROP TABLE IF EXISTS test.compression_codec_multiple_tiny_log; + +CREATE TABLE test.compression_codec_multiple_tiny_log ( + id UInt64 CODEC(LZ4, ZSTD, NONE, LZ4HC, Delta(4)), + data String CODEC(ZSTD(2), NONE, Delta(2), LZ4HC, LZ4, LZ4, Delta(8)), + ddd Date CODEC(NONE, NONE, NONE, Delta(1), LZ4, ZSTD, LZ4HC, LZ4HC), + somenum Float64 CODEC(Delta(4), LZ4, LZ4, ZSTD(2), LZ4HC(5), ZSTD(3), ZSTD) +) ENGINE = TinyLog(); + +SHOW CREATE TABLE test.compression_codec_multiple_tiny_log; + +INSERT INTO test.compression_codec_multiple_tiny_log VALUES (1, 'world', toDate('2018-10-05'), 1.1), (2, 'hello', toDate('2018-10-01'), 2.2), (3, 'buy', toDate('2018-10-11'), 3.3); + +SELECT * FROM test.compression_codec_multiple_tiny_log ORDER BY id; + +INSERT INTO test.compression_codec_multiple_tiny_log select modulo(number, 100), toString(number), toDate('2018-12-01'), 5.5 * number FROM system.numbers limit 10000; + +SELECT count(*) FROM test.compression_codec_multiple_tiny_log; + +SELECT count(distinct data) FROM test.compression_codec_multiple_tiny_log; + +SELECT floor(sum(somenum), 1) FROM test.compression_codec_multiple_tiny_log; + +TRUNCATE TABLE test.compression_codec_multiple_tiny_log; + +INSERT INTO test.compression_codec_multiple_tiny_log select modulo(number, 100), toString(number), toDate('2018-12-01'), 5.5 * number FROM system.numbers limit 10000; + +SELECT sum(cityHash64(*)) FROM test.compression_codec_multiple_tiny_log; From 88c413fbe2565cb91ec232d38504a472118e9464 Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 21 Jan 2019 17:03:34 +0300 Subject: [PATCH 07/13] Remove accidental change --- dbms/src/Storages/StorageTinyLog.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 9f87fb172fc..2d6b3983af4 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -237,7 +237,6 @@ void TinyLogBlockInputStream::readData(const String & name, const IDataType & ty IDataType::OutputStreamGetter TinyLogBlockOutputStream::createStreamGetter(const String & name, WrittenStreams & written_streams) { - return [&] (const IDataType::SubstreamPath & path) -> WriteBuffer * { String stream_name = IDataType::getFileNameForStream(name, path); From 2eb7b152734a1e2b03e71a65cc674a99d8b9b9ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 21 Jan 2019 19:14:26 +0300 Subject: [PATCH 08/13] Addition to prev. revision #4103 --- dbms/programs/client/TestHint.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dbms/programs/client/TestHint.h b/dbms/programs/client/TestHint.h index 66e86b5e750..b260bead0ba 100644 --- a/dbms/programs/client/TestHint.h +++ b/dbms/programs/client/TestHint.h @@ -5,8 +5,6 @@ #include #include #include -#include -#include #include @@ -93,11 +91,15 @@ private: void parse(const String & hint) { - ReadBufferFromString ss(hint); + std::stringstream ss; + ss << hint; + String item; + while (!ss.eof()) { - String item; ss >> item; + if (ss.eof()) + break; if (item == "serverError") ss >> server_error; From f7ecfc776b3131209b05007e54f33aeb72993975 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 21 Jan 2019 21:04:08 +0300 Subject: [PATCH 09/13] Miscellaneous #4111 --- dbms/src/Compression/CompressedWriteBuffer.cpp | 2 +- dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp | 2 +- dbms/src/Storages/StorageLog.cpp | 2 +- dbms/src/Storages/StorageTinyLog.cpp | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/src/Compression/CompressedWriteBuffer.cpp b/dbms/src/Compression/CompressedWriteBuffer.cpp index 7fc8d5ab5f9..1285949c863 100644 --- a/dbms/src/Compression/CompressedWriteBuffer.cpp +++ b/dbms/src/Compression/CompressedWriteBuffer.cpp @@ -43,7 +43,7 @@ CompressedWriteBuffer::CompressedWriteBuffer( WriteBuffer & out_, CompressionCodecPtr codec_, size_t buf_size) - : BufferWithOwnMemory(buf_size), out(out_), codec(codec_) + : BufferWithOwnMemory(buf_size), out(out_), codec(std::move(codec_)) { } diff --git a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp index ff47b97fd60..b64543124b8 100644 --- a/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/dbms/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -32,7 +32,7 @@ IMergedBlockOutputStream::IMergedBlockOutputStream( min_compress_block_size(min_compress_block_size_), max_compress_block_size(max_compress_block_size_), aio_threshold(aio_threshold_), - codec(codec_) + codec(std::move(codec_)) { } diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index 5ee8726dd52..1f2fa87da53 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -146,7 +146,7 @@ private: { Stream(const std::string & data_path, CompressionCodecPtr codec, size_t max_compress_block_size) : plain(data_path, max_compress_block_size, O_APPEND | O_CREAT | O_WRONLY), - compressed(plain, codec, max_compress_block_size) + compressed(plain, std::move(codec), max_compress_block_size) { plain_offset = Poco::File(data_path).getSize(); } diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 2d6b3983af4..c7546742202 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -137,7 +137,7 @@ private: { Stream(const std::string & data_path, CompressionCodecPtr codec, size_t max_compress_block_size) : plain(data_path, max_compress_block_size, O_APPEND | O_CREAT | O_WRONLY), - compressed(plain, codec, max_compress_block_size) + compressed(plain, std::move(codec), max_compress_block_size) { } From c70e8cc5f0b250b4c472c238d37d4418e9823bcf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 21 Jan 2019 22:45:26 +0300 Subject: [PATCH 10/13] Miscellaneous #3726 --- dbms/src/Interpreters/Cluster.cpp | 20 +++++++------------ dbms/src/Interpreters/Cluster.h | 10 +++++----- dbms/src/Interpreters/DDLWorker.cpp | 6 ++---- .../Storages/Distributed/DirectoryMonitor.cpp | 3 +-- .../DistributedBlockOutputStream.cpp | 2 +- 5 files changed, 16 insertions(+), 25 deletions(-) diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index a74286dd1d0..afeed3ba577 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -104,18 +104,17 @@ String Cluster::Address::readableString() const return res; } -void Cluster::Address::fromString(const String & host_port_string, String & host_name, UInt16 & port) +std::pair Cluster::Address::fromString(const String & host_port_string) { auto pos = host_port_string.find_last_of(':'); if (pos == std::string::npos) throw Exception("Incorrect : format " + host_port_string, ErrorCodes::SYNTAX_ERROR); - host_name = unescapeForFileName(host_port_string.substr(0, pos)); - port = parse(host_port_string.substr(pos + 1)); + return {unescapeForFileName(host_port_string.substr(0, pos)), parse(host_port_string.substr(pos + 1))}; } -String Cluster::Address::toStringFull() const +String Cluster::Address::toFullString() const { return escapeForFileName(user) + @@ -126,7 +125,7 @@ String Cluster::Address::toStringFull() const + ((secure == Protocol::Secure::Enable) ? "+secure" : ""); } -void Cluster::Address::fromFullString(const String & full_string, Cluster::Address & address) +Cluster::Address Cluster::Address::fromFullString(const String & full_string) { const char * address_begin = full_string.data(); const char * address_end = address_begin + full_string.size(); @@ -152,19 +151,14 @@ void Cluster::Address::fromFullString(const String & full_string, Cluster::Addre const char * has_db = strchr(full_string.data(), '#'); const char * port_end = has_db ? has_db : address_end; + Address address; address.secure = secure; address.port = parse(host_end + 1, port_end - (host_end + 1)); address.host_name = unescapeForFileName(std::string(user_pw_end + 1, host_end)); address.user = unescapeForFileName(std::string(address_begin, has_pw ? colon : user_pw_end)); address.password = has_pw ? unescapeForFileName(std::string(colon + 1, user_pw_end)) : std::string(); address.default_database = has_db ? unescapeForFileName(std::string(has_db + 1, address_end)) : std::string(); -} - -bool Cluster::Address::operator==(const Cluster::Address & other) const -{ - return other.host_name == host_name && other.port == port - && other.secure == secure && other.user == user - && other.password == password && other.default_database == default_database; + return address; } @@ -303,7 +297,7 @@ Cluster::Cluster(const Poco::Util::AbstractConfiguration & config, const Setting { if (internal_replication) { - auto dir_name = replica_addresses.back().toStringFull(); + auto dir_name = replica_addresses.back().toFullString(); if (first) dir_name_for_internal_replication = dir_name; else diff --git a/dbms/src/Interpreters/Cluster.h b/dbms/src/Interpreters/Cluster.h index 999f90043e9..0d2d83f7bf7 100644 --- a/dbms/src/Interpreters/Cluster.h +++ b/dbms/src/Interpreters/Cluster.h @@ -78,12 +78,11 @@ public: static String toString(const String & host_name, UInt16 port); - static void fromString(const String & host_port_string, String & host_name, UInt16 & port); + static std::pair fromString(const String & host_port_string); /// Retrurns escaped user:password@resolved_host_address:resolved_host_port#default_database - String toStringFull() const; - - static void fromFullString(const String & address_full_string, Address & address); + String toFullString() const; + static Address fromFullString(const String & address_full_string); /// Returns initially resolved address Poco::Net::SocketAddress getResolvedAddress() const @@ -91,7 +90,8 @@ public: return initially_resolved_address; } - bool operator==(const Address & other) const; + auto tuple() const { return std::tie(host_name, port, secure, user, password, default_database); } + bool operator==(const Address & other) const { return tuple() == other.tuple(); } private: Poco::Net::SocketAddress initially_resolved_address; diff --git a/dbms/src/Interpreters/DDLWorker.cpp b/dbms/src/Interpreters/DDLWorker.cpp index 54fcffbea2a..7671fcb1b9b 100644 --- a/dbms/src/Interpreters/DDLWorker.cpp +++ b/dbms/src/Interpreters/DDLWorker.cpp @@ -70,7 +70,7 @@ struct HostID static HostID fromString(const String & host_port_str) { HostID res; - Cluster::Address::fromString(host_port_str, res.host_name, res.port); + std::tie(res.host_name, res.port) = Cluster::Address::fromString(host_port_str); return res; } @@ -1076,9 +1076,7 @@ public: status.tryDeserializeText(status_data); } - String host; - UInt16 port; - Cluster::Address::fromString(host_id, host, port); + auto [host, port] = Cluster::Address::fromString(host_id); if (status.code != 0 && first_exception == nullptr) first_exception = std::make_unique("There was an error on [" + host + ":" + toString(port) + "]: " + status.message, status.code); diff --git a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp index 29a77557c6b..3cd51d0bf98 100644 --- a/dbms/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/dbms/src/Storages/Distributed/DirectoryMonitor.cpp @@ -48,8 +48,7 @@ namespace for (auto it = boost::make_split_iterator(name, boost::first_finder(",")); it != decltype(it){}; ++it) { - Cluster::Address address; - Cluster::Address::fromFullString(boost::copy_range(*it), address); + Cluster::Address address = Cluster::Address::fromFullString(boost::copy_range(*it)); pools.emplace_back(factory(address)); } diff --git a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp index 19ba8e48653..afbc7855c8f 100644 --- a/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp +++ b/dbms/src/Storages/Distributed/DistributedBlockOutputStream.cpp @@ -494,7 +494,7 @@ void DistributedBlockOutputStream::writeAsyncImpl(const Block & block, const siz std::vector dir_names; for (const auto & address : cluster->getShardsAddresses()[shard_id]) if (!address.is_local) - dir_names.push_back(address.toStringFull()); + dir_names.push_back(address.toFullString()); if (!dir_names.empty()) writeToShard(block, dir_names); From 1d0bee7deeccf59f59179708e2508d49adc8de6f Mon Sep 17 00:00:00 2001 From: chertus Date: Mon, 21 Jan 2019 22:56:11 +0300 Subject: [PATCH 11/13] fix collecting required columns in lambda --- dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp | 2 +- .../queries/0_stateless/00819_ast_refactoring_bugs.reference | 0 .../tests/queries/0_stateless/00819_ast_refactoring_bugs.sql | 5 +++++ 3 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.reference create mode 100644 dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.sql diff --git a/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp b/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp index 7d93da95e8d..5dc479fee5e 100644 --- a/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp +++ b/dbms/src/Interpreters/RequiredSourceColumnsVisitor.cpp @@ -157,7 +157,7 @@ void RequiredSourceColumnsMatcher::visit(const ASTFunction & node, const ASTPtr local_aliases.push_back(name); /// visit child with masked local aliases - visit(node.arguments->children[1], data); + RequiredSourceColumnsVisitor(data).visit(node.arguments->children[1]); for (const auto & name : local_aliases) data.private_aliases.erase(name); diff --git a/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.reference b/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.sql b/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.sql new file mode 100644 index 00000000000..98ae13d2b9e --- /dev/null +++ b/dbms/tests/queries/0_stateless/00819_ast_refactoring_bugs.sql @@ -0,0 +1,5 @@ +CREATE TABLE IF NOT EXISTS test.sign (Sign Int8, Arr Array(Int8)) ENGINE = Memory; + +SELECT arrayMap(x -> x * Sign, Arr) FROM test.sign; + +DROP TABLE test.sign; From 24fc3ad544d6f29dfd505411c3a8a4eeeb21ced8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 22 Jan 2019 02:29:09 +0300 Subject: [PATCH 12/13] Auto version update to [19.1.4] [54413] --- dbms/cmake/version.cmake | 8 ++++---- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/dbms/cmake/version.cmake b/dbms/cmake/version.cmake index ccb379a4a69..d1656c9f90b 100644 --- a/dbms/cmake/version.cmake +++ b/dbms/cmake/version.cmake @@ -2,10 +2,10 @@ set(VERSION_REVISION 54413) set(VERSION_MAJOR 19) set(VERSION_MINOR 1) -set(VERSION_PATCH 3) -set(VERSION_GITHASH ac0060079ab221278338db343ca9eaf006fc4ee1) -set(VERSION_DESCRIBE v19.1.3-testing) -set(VERSION_STRING 19.1.3) +set(VERSION_PATCH 4) +set(VERSION_GITHASH be762f58e1d0286c54609301cabc1934f49257fc) +set(VERSION_DESCRIBE v19.1.4-testing) +set(VERSION_STRING 19.1.4) # end of autochange set(VERSION_EXTRA "" CACHE STRING "") diff --git a/debian/changelog b/debian/changelog index 0b436d08615..36903d88d05 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (19.1.3) unstable; urgency=low +clickhouse (19.1.4) unstable; urgency=low * Modified source code - -- Mon, 21 Jan 2019 16:26:13 +0300 + -- Tue, 22 Jan 2019 02:29:09 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index c607bbb2f46..7cf376c6ed1 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.1.3 +ARG version=19.1.4 RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 4d40cbe7714..a45221e060e 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.1.3 +ARG version=19.1.4 ARG gosu_ver=1.10 RUN apt-get update \ diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index c7b325e4d58..2a21d932422 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb http://repo.yandex.ru/clickhouse/deb/stable/ main/" -ARG version=19.1.3 +ARG version=19.1.4 RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From a9054e05f05ccf58416dc091002313db3c814d2a Mon Sep 17 00:00:00 2001 From: ogorbacheva Date: Tue, 22 Jan 2019 16:21:20 +0300 Subject: [PATCH 13/13] Doc fix: edit info about escaping (#4118) * Doc fix: delete VerticalRaw format and remove escaping from Pretty and Vertical formats * Doc fix: editing text about escaping * Doc fix: fix example in pretty format * Doc fix: fix example in pretty format * Doc fix: last fixes * Doc fix: last fixes * Doc fix --- docs/en/interfaces/formats.md | 48 ++++++++++++++++------------------- 1 file changed, 22 insertions(+), 26 deletions(-) diff --git a/docs/en/interfaces/formats.md b/docs/en/interfaces/formats.md index c47e75142df..eddefaa9394 100644 --- a/docs/en/interfaces/formats.md +++ b/docs/en/interfaces/formats.md @@ -14,7 +14,6 @@ The table below lists supported formats and how they can be used in `INSERT` and | [CSVWithNames](#csvwithnames) | ✔ | ✔ | | [Values](#values) | ✔ | ✔ | | [Vertical](#vertical) | ✗ | ✔ | -| [VerticalRaw](#verticalraw) | ✗ | ✔ | | [JSON](#json) | ✗ | ✔ | | [JSONCompact](#jsoncompact) | ✗ | ✔ | | [JSONEachRow](#jsoneachrow) | ✔ | ✔ | @@ -345,6 +344,8 @@ Each result block is output as a separate table. This is necessary so that block [NULL](../query_language/syntax.md) is output as `ᴺᵁᴸᴸ`. +Example (shown for the [PrettyCompact](#prettycompact) format): + ``` sql SELECT * FROM t_null ``` @@ -355,10 +356,22 @@ SELECT * FROM t_null └───┴──────┘ ``` +Rows are not escaped in Pretty* formats. Example is shown for the [PrettyCompact](#prettycompact) format: + +``` sql +SELECT 'String with \'quotes\' and \t character' AS Escaping_test +``` + +``` +┌─Escaping_test────────────────────────┐ +│ String with 'quotes' and character │ +└──────────────────────────────────────┘ +``` + To avoid dumping too much data to the terminal, only the first 10,000 rows are printed. If the number of rows is greater than or equal to 10,000, the message "Showed first 10 000" is printed. This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). -The Pretty format supports outputting total values (when using WITH TOTALS) and extremes (when 'extremes' is set to 1). In these cases, total values and extreme values are output after the main data, in separate tables. Example (shown for the PrettyCompact format): +The Pretty format supports outputting total values (when using WITH TOTALS) and extremes (when 'extremes' is set to 1). In these cases, total values and extreme values are output after the main data, in separate tables. Example (shown for the [PrettyCompact](#prettycompact) format): ``` sql SELECT EventDate, count() AS c FROM test.hits GROUP BY EventDate WITH TOTALS ORDER BY EventDate FORMAT PrettyCompact @@ -389,7 +402,7 @@ Extremes: ## PrettyCompact {#prettycompact} -Differs from `Pretty` in that the grid is drawn between rows and the result is more compact. +Differs from [Pretty](#pretty) in that the grid is drawn between rows and the result is more compact. This format is used by default in the command-line client in interactive mode. ## PrettyCompactMonoBlock {#prettycompactmonoblock} @@ -461,37 +474,20 @@ Row 1: x: 1 y: ᴺᵁᴸᴸ ``` +Rows are not escaped in Vertical format: -This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). - -## VerticalRaw {#verticalraw} - -Differs from `Vertical` format in that the rows are not escaped. -This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). - -Examples: +``` sql +SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical +``` ``` -:) SHOW CREATE TABLE geonames FORMAT VerticalRaw; Row 1: ────── -statement: CREATE TABLE default.geonames ( geonameid UInt32, date Date DEFAULT CAST('2017-12-08' AS Date)) ENGINE = MergeTree(date, geonameid, 8192) - -:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT VerticalRaw; -Row 1: -────── -test: string with 'quotes' and with some special +test: string with 'quotes' and with some special characters ``` -Compare with the Vertical format: - -``` -:) SELECT 'string with \'quotes\' and \t with some special \n characters' AS test FORMAT Vertical; -Row 1: -────── -test: string with \'quotes\' and \t with some special \n characters -``` +This format is only appropriate for outputting a query result, but not for parsing (retrieving data to insert in a table). ## XML {#xml}