From 9880c628393ae52d82d774ce5f88f5f0f13c2b7b Mon Sep 17 00:00:00 2001 From: Evgeniy Gatov Date: Wed, 22 Apr 2015 20:56:27 +0300 Subject: [PATCH 001/109] libzkutil: tiny [#METR-13470]. --- libs/libzkutil/include/zkutil/Increment.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/libs/libzkutil/include/zkutil/Increment.h b/libs/libzkutil/include/zkutil/Increment.h index 5cd8f02316c..37b698d0290 100644 --- a/libs/libzkutil/include/zkutil/Increment.h +++ b/libs/libzkutil/include/zkutil/Increment.h @@ -2,13 +2,13 @@ #include -namespace zkutil +namespace zkutil { class Increment { public: - Increment(ZooKeeperPtr zk_, const std::string & path_) + Increment(ZooKeeperPtr zk_, const std::string & path_) : zk(zk_), path(path_) { zk->createAncestors(path); @@ -34,8 +34,8 @@ public: { success = zk->tryCreate(path, std::to_string(result), zkutil::CreateMode::Persistent) == ZOK; } - } - while(!success); + } + while (!success); return result; } @@ -45,4 +45,4 @@ private: Logger * log = &Logger::get("zkutil::Increment"); }; -} \ No newline at end of file +} From 728d367d342c29f3d9f00b9ac9faedd505987a6e Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 21 May 2015 14:56:47 +0300 Subject: [PATCH 002/109] dbms: FunctionsMath: use Vec2d::store for faster stores --- dbms/include/DB/Functions/FunctionsMath.h | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsMath.h b/dbms/include/DB/Functions/FunctionsMath.h index de6a7893bdb..2e1518079a9 100644 --- a/dbms/include/DB/Functions/FunctionsMath.h +++ b/dbms/include/DB/Functions/FunctionsMath.h @@ -165,9 +165,8 @@ struct UnaryFunctionVectorized template static void execute(const T * const src, Float64 * const dst) { - const auto & result = Function(Vec2d(src[0], src[1])); - dst[0] = result[0]; - dst[1] = result[1]; + const auto result = Function(Vec2d(src[0], src[1])); + result.store(dst); } }; @@ -432,9 +431,8 @@ struct BinaryFunctionVectorized template static void execute(const T1 * const src_left, const T2 * const src_right, Float64 * const dst) { - const auto & result = Function(Vec2d(src_left[0], src_left[1]), Vec2d(src_right[0], src_right[1])); - dst[0] = result[0]; - dst[1] = result[1]; + const auto result = Function(Vec2d(src_left[0], src_left[1]), Vec2d(src_right[0], src_right[1])); + result.store(dst); } }; From 8ddc49125c376748fd20571106df4297acf3927b Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 21 May 2015 16:47:29 +0300 Subject: [PATCH 003/109] dbms: do not expand ALIAS column names. [#METR-16390] --- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 8dba64da073..07a8fc7d1e9 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -281,7 +281,7 @@ void ExpressionAnalyzer::addStorageAliases() return; for (const auto & alias : storage->alias_columns) - aliases[alias.name] = storage->column_defaults[alias.name].expression; + (aliases[alias.name] = storage->column_defaults[alias.name].expression->clone())->setAlias(alias.name); } From 9b5ff37c09ebc0a90ac6213b4e9e84251e17ac14 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 21 May 2015 19:26:23 +0300 Subject: [PATCH 004/109] dbms: set optimize_move_to_prewhere to on by default [#METR-13020] --- dbms/include/DB/Interpreters/Settings.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/include/DB/Interpreters/Settings.h b/dbms/include/DB/Interpreters/Settings.h index 4310733c205..32449c72086 100644 --- a/dbms/include/DB/Interpreters/Settings.h +++ b/dbms/include/DB/Interpreters/Settings.h @@ -72,7 +72,7 @@ struct Settings M(SettingMilliseconds, distributed_directory_monitor_sleep_time_ms, DBMS_DISTRIBUTED_DIRECTORY_MONITOR_SLEEP_TIME_MS) \ \ /** Allows disabling WHERE to PREWHERE optimization in SELECT queries from MergeTree */ \ - M(SettingBool, optimize_move_to_prewhere, false) \ + M(SettingBool, optimize_move_to_prewhere, true) \ \ /** Ожидать выполнения действий по манипуляции с партициями. 0 - не ждать, 1 - ждать выполнения только у себя, 2 - ждать всех. */ \ M(SettingUInt64, replication_alter_partitions_sync, 1) \ From 3fc0fda58ffb729b5d86c7856bf36e9d1632bcff Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 21 May 2015 19:34:08 +0300 Subject: [PATCH 005/109] dbms: ExpressionAnalyzer: addStorageAliases add useful todo --- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 07a8fc7d1e9..593fa39b991 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -280,6 +280,7 @@ void ExpressionAnalyzer::addStorageAliases() if (!storage) return; + /// @todo: consider storing default expressions with alias set to avoid cloning for (const auto & alias : storage->alias_columns) (aliases[alias.name] = storage->column_defaults[alias.name].expression->clone())->setAlias(alias.name); } From f465feeb289b293110f94b3beea292f3bf6d5615 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 22 May 2015 16:25:45 +0300 Subject: [PATCH 006/109] dbms: DictionarySource::loadIds pass ids by reference to const. Allow specifying "where" for MySQL and ClickHouse dictionary sources [#METR-16432] --- .../Dictionaries/ClickHouseDictionarySource.h | 33 +++++++++++++---- .../DB/Dictionaries/FileDictionarySource.h | 2 +- .../DB/Dictionaries/IDictionarySource.h | 2 +- .../DB/Dictionaries/MySQLDictionarySource.h | 35 +++++++++++++++---- 4 files changed, 57 insertions(+), 15 deletions(-) diff --git a/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h b/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h index 0d08a70ea10..56671aa774c 100644 --- a/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h +++ b/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h @@ -29,19 +29,21 @@ public: password{config.getString(config_prefix + ".password", "")}, db{config.getString(config_prefix + ".db", "")}, table{config.getString(config_prefix + ".table")}, + where{config.getString(config_prefix + ".where", "")}, sample_block{sample_block}, context(context), is_local{isLocalAddress({ host, port })}, pool{is_local ? nullptr : std::make_unique( max_connections, host, port, db, user, password, context.getDataTypeFactory(), "ClickHouseDictionarySource") }, - load_all_query{composeLoadAllQuery(sample_block, db, table)} + load_all_query{composeLoadAllQuery()} {} /// copy-constructor is provided in order to support cloneability ClickHouseDictionarySource(const ClickHouseDictionarySource & other) : host{other.host}, port{other.port}, user{other.user}, password{other.password}, db{other.db}, table{other.table}, + where{other.where}, sample_block{other.sample_block}, context(other.context), is_local{other.is_local}, pool{is_local ? nullptr : std::make_unique( @@ -60,7 +62,7 @@ public: return new RemoteBlockInputStream{pool.get(), load_all_query, nullptr}; } - BlockInputStreamPtr loadIds(const std::vector ids) override + BlockInputStreamPtr loadIds(const std::vector & ids) override { const auto query = composeLoadIdsQuery(ids); @@ -74,10 +76,13 @@ public: DictionarySourcePtr clone() const override { return std::make_unique(*this); } - std::string toString() const override { return "ClickHouse: " + db + '.' + table; } + std::string toString() const override + { + return "ClickHouse: " + db + '.' + table + (where.empty() ? "" : ", where: " + where); + } private: - static std::string composeLoadAllQuery(const Block & block, const std::string & db, const std::string & table) + std::string composeLoadAllQuery() const { std::string query; @@ -86,12 +91,12 @@ private: writeString("SELECT ", out); auto first = true; - for (const auto idx : ext::range(0, block.columns())) + for (const auto idx : ext::range(0, sample_block.columns())) { if (!first) writeString(", ", out); - writeString(block.getByPosition(idx).name, out); + writeString(sample_block.getByPosition(idx).name, out); first = false; } @@ -102,6 +107,13 @@ private: writeChar('.', out); } writeProbablyBackQuotedString(table, out); + + if (!where.empty()) + { + writeString(" WHERE ", out); + writeString(where, out); + } + writeChar(';', out); } @@ -134,7 +146,15 @@ private: writeChar('.', out); } writeProbablyBackQuotedString(table, out); + writeString(" WHERE ", out); + + if (!where.empty()) + { + writeString(where, out); + writeString(" AND ", out); + } + writeProbablyBackQuotedString(id_column_name, out); writeString(" IN (", out); @@ -160,6 +180,7 @@ private: const std::string password; const std::string db; const std::string table; + const std::string where; Block sample_block; Context & context; const bool is_local; diff --git a/dbms/include/DB/Dictionaries/FileDictionarySource.h b/dbms/include/DB/Dictionaries/FileDictionarySource.h index 1845652a52b..cadb3fb085c 100644 --- a/dbms/include/DB/Dictionaries/FileDictionarySource.h +++ b/dbms/include/DB/Dictionaries/FileDictionarySource.h @@ -40,7 +40,7 @@ public: return new OwningBufferBlockInputStream{stream, std::move(in_ptr)}; } - BlockInputStreamPtr loadIds(const std::vector ids) override + BlockInputStreamPtr loadIds(const std::vector & ids) override { throw Exception{ "Method unsupported", diff --git a/dbms/include/DB/Dictionaries/IDictionarySource.h b/dbms/include/DB/Dictionaries/IDictionarySource.h index 1a528e50bb3..ad3f70f6003 100644 --- a/dbms/include/DB/Dictionaries/IDictionarySource.h +++ b/dbms/include/DB/Dictionaries/IDictionarySource.h @@ -25,7 +25,7 @@ public: virtual bool supportsSelectiveLoad() const = 0; /// returns an input stream with the data for a collection of identifiers - virtual BlockInputStreamPtr loadIds(const std::vector ids) = 0; + virtual BlockInputStreamPtr loadIds(const std::vector & ids) = 0; /// indicates whether the source has been modified since last load* operation virtual bool isModified() const = 0; diff --git a/dbms/include/DB/Dictionaries/MySQLDictionarySource.h b/dbms/include/DB/Dictionaries/MySQLDictionarySource.h index f74105b8ea3..f47ce818102 100644 --- a/dbms/include/DB/Dictionaries/MySQLDictionarySource.h +++ b/dbms/include/DB/Dictionaries/MySQLDictionarySource.h @@ -20,9 +20,10 @@ public: Block & sample_block) : db{config.getString(config_prefix + ".db", "")}, table{config.getString(config_prefix + ".table")}, + where{config.getString(config_prefix + ".where", "")}, sample_block{sample_block}, pool{config, config_prefix}, - load_all_query{composeLoadAllQuery(sample_block, db, table)}, + load_all_query{composeLoadAllQuery()}, last_modification{getLastModification()} {} @@ -30,6 +31,7 @@ public: MySQLDictionarySource(const MySQLDictionarySource & other) : db{other.db}, table{other.table}, + where{other.where}, sample_block{other.sample_block}, pool{other.pool}, load_all_query{other.load_all_query}, last_modification{other.last_modification} @@ -41,7 +43,7 @@ public: return new MySQLBlockInputStream{pool.Get(), load_all_query, sample_block, max_block_size}; } - BlockInputStreamPtr loadIds(const std::vector ids) override + BlockInputStreamPtr loadIds(const std::vector & ids) override { last_modification = getLastModification(); const auto query = composeLoadIdsQuery(ids); @@ -54,7 +56,10 @@ public: DictionarySourcePtr clone() const override { return std::make_unique(*this); } - std::string toString() const override { return "MySQL: " + db + '.' + table; } + std::string toString() const override + { + return "MySQL: " + db + '.' + table + (where.empty() ? "" : ", where: " + where); + } private: mysqlxx::DateTime getLastModification() const @@ -88,7 +93,7 @@ private: return update_time; } - static std::string composeLoadAllQuery(const Block & block, const std::string & db, const std::string & table) + std::string composeLoadAllQuery() const { std::string query; @@ -97,12 +102,12 @@ private: writeString("SELECT ", out); auto first = true; - for (const auto idx : ext::range(0, block.columns())) + for (const auto idx : ext::range(0, sample_block.columns())) { if (!first) writeString(", ", out); - writeString(block.getByPosition(idx).name, out); + writeString(sample_block.getByPosition(idx).name, out); first = false; } @@ -113,13 +118,20 @@ private: writeChar('.', out); } writeProbablyBackQuotedString(table, out); + + if (!where.empty()) + { + writeString(" WHERE ", out); + writeString(where, out); + } + writeChar(';', out); } return query; } - std::string composeLoadIdsQuery(const std::vector ids) + std::string composeLoadIdsQuery(const std::vector & ids) { std::string query; @@ -145,7 +157,15 @@ private: writeChar('.', out); } writeProbablyBackQuotedString(table, out); + writeString(" WHERE ", out); + + if (!where.empty()) + { + writeString(where, out); + writeString(" AND ", out); + } + writeProbablyBackQuotedString(id_column_name, out); writeString(" IN (", out); @@ -167,6 +187,7 @@ private: const std::string db; const std::string table; + const std::string where; Block sample_block; mutable mysqlxx::PoolWithFailover pool; const std::string load_all_query; From 65d74a61952f9631dd795fdc7afcdfa2a9724428 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 21 May 2015 12:38:02 +0300 Subject: [PATCH 007/109] dbms: add network compression method to settings [#METR-16391] dbms: add network compression method to settings [#METR-16391] --- dbms/include/DB/IO/CompressedStream.h | 8 +-- dbms/include/DB/Interpreters/Settings.h | 3 + dbms/include/DB/Interpreters/SettingsCommon.h | 63 ++++++++++++++++++- 3 files changed, 69 insertions(+), 5 deletions(-) diff --git a/dbms/include/DB/IO/CompressedStream.h b/dbms/include/DB/IO/CompressedStream.h index 39364881c65..c12f33f2a29 100644 --- a/dbms/include/DB/IO/CompressedStream.h +++ b/dbms/include/DB/IO/CompressedStream.h @@ -14,10 +14,10 @@ namespace DB /** Метод сжатия */ enum class CompressionMethod { - QuickLZ, - LZ4, - LZ4HC, /// Формат такой же, как у LZ4. Разница только при сжатии. - ZSTD, /// Экспериментальный алгоритм: https://github.com/Cyan4973/zstd + QuickLZ = 0, + LZ4 = 1, + LZ4HC = 2, /// Формат такой же, как у LZ4. Разница только при сжатии. + ZSTD = 3, /// Экспериментальный алгоритм: https://github.com/Cyan4973/zstd }; /** Формат сжатого блока следующий: diff --git a/dbms/include/DB/Interpreters/Settings.h b/dbms/include/DB/Interpreters/Settings.h index 32449c72086..09839370cb0 100644 --- a/dbms/include/DB/Interpreters/Settings.h +++ b/dbms/include/DB/Interpreters/Settings.h @@ -129,6 +129,9 @@ struct Settings * но чтобы каждый источник динамически выбирал себе доступную работу. \ */ \ M(SettingFloat, max_streams_to_max_threads_ratio, 1) \ + \ + /** Позволяет выбирать метод сжатия данных при записи */\ + M(SettingNetworkCompressionMethod, network_compression_method, CompressionMethod::LZ4) \ /// Всевозможные ограничения на выполнение запроса. Limits limits; diff --git a/dbms/include/DB/Interpreters/SettingsCommon.h b/dbms/include/DB/Interpreters/SettingsCommon.h index 5403395070d..0eb5f06974d 100644 --- a/dbms/include/DB/Interpreters/SettingsCommon.h +++ b/dbms/include/DB/Interpreters/SettingsCommon.h @@ -4,7 +4,7 @@ #include #include #include - +#include namespace DB { @@ -534,5 +534,66 @@ struct SettingOverflowMode } }; +struct SettingNetworkCompressionMethod +{ + CompressionMethod value; + bool changed = false; + SettingNetworkCompressionMethod(CompressionMethod x = CompressionMethod::LZ4) : value(x) {} + + operator CompressionMethod() const { return value; } + SettingNetworkCompressionMethod & operator= (CompressionMethod x) { set(x); return *this; } + + static CompressionMethod getCompressionMethod(const String & s) + { + if (s == "quicklz") + return CompressionMethod::QuickLZ; + if (s == "lz4") + return CompressionMethod::LZ4; + if (s == "lz4hc") + return CompressionMethod::LZ4HC; + if (s == "zstd") + return CompressionMethod::ZSTD; + + throw Exception("Unknown network compression method: '" + s + "', must be one of 'quicklz', 'lz4', 'lz4hc', 'zstd' ", ErrorCodes::UNKNOWN_COMPRESSION_METHOD); + } + + String toString() const + { + const char * strings[] = { "quicklz", "lz4", "lz4hc", "zstd" }; + + if (value < CompressionMethod::QuickLZ || value > CompressionMethod::ZSTD) + throw Exception("Unknown compression method", ErrorCodes::UNKNOWN_COMPRESSION_METHOD); + + return strings[static_cast(value)]; + } + + void set(CompressionMethod x) + { + value = x; + changed = true; + } + + void set(const Field & x) + { + set(safeGet(x)); + } + + void set(const String & x) + { + set(getCompressionMethod(x)); + } + + void set(ReadBuffer & buf) + { + String x; + readBinary(x, buf); + set(x); + } + + void write(WriteBuffer & buf) const + { + writeBinary(toString(), buf); + } +}; } From 391ec6451bdfa6c93045c3e58751ae1296e1403b Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 21 May 2015 14:48:10 +0300 Subject: [PATCH 008/109] dbms: use network_compression_method when server writes result [#METR-16391] --- dbms/src/Server/TCPHandler.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp index 9bbcaa0a3aa..1961e0c0b9b 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/src/Server/TCPHandler.cpp @@ -617,7 +617,7 @@ void TCPHandler::initBlockOutput() if (!state.block_out) { if (state.compression == Protocol::Compression::Enable) - state.maybe_compressed_out = new CompressedWriteBuffer(*out); + state.maybe_compressed_out = new CompressedWriteBuffer(*out, query_context.getSettings().network_compression_method); else state.maybe_compressed_out = out; From 392dc138ba5401a80d38bb9020eaaed5b3486b59 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 21 May 2015 15:48:45 +0300 Subject: [PATCH 009/109] dbms: add compression method to client [#METR-16391] --- dbms/include/DB/Client/Connection.h | 1 + dbms/src/Client/Connection.cpp | 4 +++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/dbms/include/DB/Client/Connection.h b/dbms/include/DB/Client/Connection.h index f1994178fdc..6a40e90e37f 100644 --- a/dbms/include/DB/Client/Connection.h +++ b/dbms/include/DB/Client/Connection.h @@ -169,6 +169,7 @@ private: String query_id; UInt64 compression; /// Сжимать ли данные при взаимодействии с сервером. + CompressionMethod network_compression_method = CompressionMethod::LZ4; /// каким алгоритмом сжимать данные const DataTypeFactory & data_type_factory; diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 91391a297e9..b177ee89638 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -236,6 +236,8 @@ bool Connection::ping() void Connection::sendQuery(const String & query, const String & query_id_, UInt64 stage, const Settings * settings, bool with_pending_data) { + network_compression_method = settings ? settings->network_compression_method : CompressionMethod::LZ4; + forceConnected(); query_id = query_id_; @@ -293,7 +295,7 @@ void Connection::sendData(const Block & block, const String & name) if (!block_out) { if (compression == Protocol::Compression::Enable) - maybe_compressed_out = new CompressedWriteBuffer(*out); + maybe_compressed_out = new CompressedWriteBuffer(*out, network_compression_method); else maybe_compressed_out = out; From 34f1b2e8a89878f45e7efd14e5422b9e01919976 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 21 May 2015 17:25:19 +0300 Subject: [PATCH 010/109] dbms: build fix [#METR-16391] --- dbms/src/Client/Connection.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index b177ee89638..9bf7de70ebf 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -236,7 +236,7 @@ bool Connection::ping() void Connection::sendQuery(const String & query, const String & query_id_, UInt64 stage, const Settings * settings, bool with_pending_data) { - network_compression_method = settings ? settings->network_compression_method : CompressionMethod::LZ4; + network_compression_method = settings ? settings->network_compression_method.value : CompressionMethod::LZ4; forceConnected(); From 665afc25505bd76126fcbf0d0b44865f845a83ad Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Thu, 21 May 2015 19:05:02 +0300 Subject: [PATCH 011/109] dbms: renamed SettingsNetworkCompressionMethod to SettingsCompressionMethod[#METR-16391] --- dbms/include/DB/Interpreters/Settings.h | 2 +- dbms/include/DB/Interpreters/SettingsCommon.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbms/include/DB/Interpreters/Settings.h b/dbms/include/DB/Interpreters/Settings.h index 09839370cb0..e178e784936 100644 --- a/dbms/include/DB/Interpreters/Settings.h +++ b/dbms/include/DB/Interpreters/Settings.h @@ -131,7 +131,7 @@ struct Settings M(SettingFloat, max_streams_to_max_threads_ratio, 1) \ \ /** Позволяет выбирать метод сжатия данных при записи */\ - M(SettingNetworkCompressionMethod, network_compression_method, CompressionMethod::LZ4) \ + M(SettingCompressionMethod, network_compression_method, CompressionMethod::LZ4) \ /// Всевозможные ограничения на выполнение запроса. Limits limits; diff --git a/dbms/include/DB/Interpreters/SettingsCommon.h b/dbms/include/DB/Interpreters/SettingsCommon.h index 0eb5f06974d..69d6ce1173a 100644 --- a/dbms/include/DB/Interpreters/SettingsCommon.h +++ b/dbms/include/DB/Interpreters/SettingsCommon.h @@ -534,15 +534,15 @@ struct SettingOverflowMode } }; -struct SettingNetworkCompressionMethod +struct SettingCompressionMethod { CompressionMethod value; bool changed = false; - SettingNetworkCompressionMethod(CompressionMethod x = CompressionMethod::LZ4) : value(x) {} + SettingCompressionMethod(CompressionMethod x = CompressionMethod::LZ4) : value(x) {} operator CompressionMethod() const { return value; } - SettingNetworkCompressionMethod & operator= (CompressionMethod x) { set(x); return *this; } + SettingCompressionMethod & operator= (CompressionMethod x) { set(x); return *this; } static CompressionMethod getCompressionMethod(const String & s) { From ce8e182d9c96cf93ef93627c62cbe0645715c656 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Fri, 22 May 2015 12:48:49 +0300 Subject: [PATCH 012/109] dbms: fix comments for network_compression_method [#METR-16391] --- dbms/include/DB/Client/Connection.h | 3 ++- dbms/include/DB/Interpreters/SettingsCommon.h | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/dbms/include/DB/Client/Connection.h b/dbms/include/DB/Client/Connection.h index 6a40e90e37f..9f81b5a7feb 100644 --- a/dbms/include/DB/Client/Connection.h +++ b/dbms/include/DB/Client/Connection.h @@ -169,7 +169,8 @@ private: String query_id; UInt64 compression; /// Сжимать ли данные при взаимодействии с сервером. - CompressionMethod network_compression_method = CompressionMethod::LZ4; /// каким алгоритмом сжимать данные + /// каким алгоритмом сжимать данные при INSERT и данные внешних таблиц + CompressionMethod network_compression_method = CompressionMethod::LZ4; const DataTypeFactory & data_type_factory; diff --git a/dbms/include/DB/Interpreters/SettingsCommon.h b/dbms/include/DB/Interpreters/SettingsCommon.h index 69d6ce1173a..63c52dc9663 100644 --- a/dbms/include/DB/Interpreters/SettingsCommon.h +++ b/dbms/include/DB/Interpreters/SettingsCommon.h @@ -555,7 +555,7 @@ struct SettingCompressionMethod if (s == "zstd") return CompressionMethod::ZSTD; - throw Exception("Unknown network compression method: '" + s + "', must be one of 'quicklz', 'lz4', 'lz4hc', 'zstd' ", ErrorCodes::UNKNOWN_COMPRESSION_METHOD); + throw Exception("Unknown compression method: '" + s + "', must be one of 'quicklz', 'lz4', 'lz4hc', 'zstd' ", ErrorCodes::UNKNOWN_COMPRESSION_METHOD); } String toString() const From 52b004722eeca8b5b300f7b86736a6b3ea4867d1 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Mon, 25 May 2015 16:20:17 +0300 Subject: [PATCH 013/109] CalcCloud: fix compability with java StoreInfo --- dbms/include/DB/Core/Defines.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/include/DB/Core/Defines.h b/dbms/include/DB/Core/Defines.h index 3d9a485fec8..ba89beee564 100644 --- a/dbms/include/DB/Core/Defines.h +++ b/dbms/include/DB/Core/Defines.h @@ -7,6 +7,7 @@ #define DBMS_DEFAULT_HOST "localhost" #define DBMS_DEFAULT_PORT 9000 #define DBMS_DEFAULT_PORT_STR "9000" +#define DBMS_DEFAULT_HTTP_PORT 8123 #define DBMS_DEFAULT_CONNECT_TIMEOUT_SEC 10 #define DBMS_DEFAULT_CONNECT_TIMEOUT_WITH_FAILOVER_MS 50 #define DBMS_DEFAULT_SEND_TIMEOUT_SEC 300 From 46529f57d9f0bf9a374e0e8cce209fc2056acdd9 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 25 May 2015 16:35:04 +0300 Subject: [PATCH 014/109] dbms: Server: Implemented MySQL-conformant behavior. [#METR-15210] --- dbms/include/DB/Functions/FunctionsRound.h | 684 +++++++++++++++------ 1 file changed, 491 insertions(+), 193 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index ffe2769f24e..c01faab598f 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -5,6 +5,7 @@ #include #include +#include namespace DB { @@ -13,9 +14,14 @@ namespace DB * roundToExp2 - вниз до ближайшей степени двойки; * roundDuration - вниз до ближайшего из: 0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000; * roundAge - вниз до ближайшего из: 0, 18, 25, 35, 45. - * round(x, N) - арифметическое округление (N - сколько знаков после запятой оставить; 0 по умолчанию). - * ceil(x, N) - наименьшее число, которое не меньше x (N - сколько знаков после запятой оставить; 0 по умолчанию). - * floor(x, N) - наибольшее число, которое не больше x (N - сколько знаков после запятой оставить; 0 по умолчанию). + * round(x, N) - арифметическое округление (N = 0 по умолчанию). + * ceil(x, N) - наименьшее число, которое не меньше x (N = 0 по умолчанию). + * floor(x, N) - наибольшее число, которое не больше x (N = 0 по умолчанию). + * + * Значение параметра N: + * - N > 0: округлять до числа с N десятичными знаками после запятой + * - N < 0: окурглять до целого числа с N нулевыми знаками + * - N = 0: округлять до целого числа */ template @@ -92,125 +98,275 @@ namespace DB } }; - /// Реализация функций округления на низком уровне. + /** Этот параметр контролирует поведение функций округления. + */ + enum ScaleMode + { + PositiveScale, // округлять до числа с N десятичными знаками после запятой + NegativeScale, // окурглять до целого числа с N нулевыми знаками + ZeroScale, // округлять до целого числа + NullScale // возвращать нулевое значение + }; - template - struct RoundingComputation + /** Реализация низкоуровневых функций округления для целочисленных значений. + */ + template + struct IntegerRoundingComputation { }; - template - struct RoundingComputation + template + struct IntegerRoundingComputation::value + && ((scale_mode == PositiveScale) || (scale_mode == ZeroScale))>::type> { - using Data = std::array; - using Scale = __m128; + static inline T compute(const T in, size_t scale) + { + return in; + } + }; - template - static inline void prepareScale(size_t scale, Scale & mm_scale, - typename std::enable_if::type * = nullptr) + template + struct IntegerRoundingComputation::value>::type> + { + static inline T compute(T in, size_t scale) + { + T rem = in % scale; + if (rem == in) + return 0; + + in -= rem; + if (static_cast(2 * rem) < scale) + return in; + else + return in + scale; + } + }; + + template + struct IntegerRoundingComputation::value>::type> + { + static inline T compute(const T in, size_t scale) + { + T rem = in % scale; + if (rem == in) + return 0; + + return in - rem + scale; + } + }; + + template + struct IntegerRoundingComputation::value>::type> + { + static inline T compute(const T in, size_t scale) + { + T rem = in % scale; + if (rem == in) + return 0; + + return in - rem; + } + }; + + template + struct BaseFloatRoundingComputation + { + }; + + template<> + struct BaseFloatRoundingComputation + { + using Scale = __m128; + static const size_t data_count = 4; + }; + + template<> + struct BaseFloatRoundingComputation + { + using Scale = __m128d; + static const size_t data_count = 2; + }; + + /** Реализация низкоуровневых функций округления для значений с плавающей точкой. + */ + template + struct FloatRoundingComputation : public BaseFloatRoundingComputation + { + }; + + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) { Float32 fscale = static_cast(scale); mm_scale = _mm_load1_ps(&fscale); } - template - static inline void prepareScale(size_t scale, Scale & mm_scale, - typename std::enable_if::type * = nullptr) + static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) { - } - - template - static inline void compute(const Data & in, const Scale & mm_scale, Data & out, - typename std::enable_if::type * = nullptr) - { - __m128 mm_value = _mm_loadu_ps(reinterpret_cast(&in)); - mm_value = _mm_mul_ps(mm_value, mm_scale); - mm_value = _mm_round_ps(mm_value, rounding_mode); - mm_value = _mm_div_ps(mm_value, mm_scale); - _mm_storeu_ps(reinterpret_cast(&out), mm_value); - } - - template - static inline void compute(const Data & in, const Scale & mm_scale, Data & out, - typename std::enable_if::type * = nullptr) - { - __m128 mm_value = _mm_loadu_ps(reinterpret_cast(&in)); - mm_value = _mm_round_ps(mm_value, rounding_mode); - _mm_storeu_ps(reinterpret_cast(&out), mm_value); + __m128 val = _mm_loadu_ps(in); + val = _mm_mul_ps(val, scale); + val = _mm_round_ps(val, rounding_mode); + val = _mm_div_ps(val, scale); + _mm_storeu_ps(out, val); } }; - template - struct RoundingComputation + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation { - using Data = std::array; - using Scale = __m128d; + static inline void prepareScale(size_t scale, Scale & mm_scale) + { + Float32 fscale = static_cast(scale); + mm_scale = _mm_load1_ps(&fscale); + } - template - static inline void prepareScale(size_t scale, Scale & mm_scale, - typename std::enable_if::type * = nullptr) + static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) + { + __m128 val = _mm_loadu_ps(in); + val = _mm_div_ps(val, scale); + __m128 res = _mm_cmpge_ps(val, getOne()); + val = _mm_round_ps(val, rounding_mode); + val = _mm_mul_ps(val, scale); + val = _mm_and_ps(val, res); + _mm_storeu_ps(out, val); + } + + private: + static inline const __m128 & getOne() + { + static const __m128 one = _mm_set1_ps(1.0); + return one; + } + }; + + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) + { + } + + static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) + { + __m128 val = _mm_loadu_ps(in); + val = _mm_round_ps(val, rounding_mode); + _mm_storeu_ps(out, val); + } + }; + + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) { Float64 fscale = static_cast(scale); mm_scale = _mm_load1_pd(&fscale); } - template - static inline void prepareScale(size_t scale, Scale & mm_scale, - typename std::enable_if::type * = nullptr) + static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) { - } - - template - static inline void compute(const Data & in, const Scale & mm_scale, Data & out, - typename std::enable_if::type * = nullptr) - { - __m128d mm_value = _mm_loadu_pd(reinterpret_cast(&in)); - mm_value = _mm_mul_pd(mm_value, mm_scale); - mm_value = _mm_round_pd(mm_value, rounding_mode); - mm_value = _mm_div_pd(mm_value, mm_scale); - _mm_storeu_pd(reinterpret_cast(&out), mm_value); - } - - template - static inline void compute(const Data & in, const Scale & mm_scale, Data & out, - typename std::enable_if::type * = nullptr) - { - __m128d mm_value = _mm_loadu_pd(reinterpret_cast(&in)); - mm_value = _mm_round_pd(mm_value, rounding_mode); - _mm_storeu_pd(reinterpret_cast(&out), mm_value); + __m128d val = _mm_loadu_pd(in); + val = _mm_mul_pd(val, scale); + val = _mm_round_pd(val, rounding_mode); + val = _mm_div_pd(val, scale); + _mm_storeu_pd(out, val); } }; - /// Реализация функций округления на высоком уровне. + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) + { + Float64 fscale = static_cast(scale); + mm_scale = _mm_load1_pd(&fscale); + } - template + static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) + { + __m128d val = _mm_loadu_pd(in); + val = _mm_div_pd(val, scale); + __m128d res = _mm_cmpge_pd(val, getOne()); + val = _mm_round_pd(val, rounding_mode); + val = _mm_mul_pd(val, scale); + val = _mm_and_pd(val, res); + _mm_storeu_pd(out, val); + } + + private: + static inline const __m128d & getOne() + { + static const __m128d one = _mm_set1_pd(1.0); + return one; + } + }; + + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) + { + } + + static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) + { + __m128d val = _mm_loadu_pd(in); + val = _mm_round_pd(val, rounding_mode); + _mm_storeu_pd(out, val); + } + }; + + /** Реализация высокоуровневых функций округления. + */ + template struct FunctionRoundingImpl { }; - /// В случае целочисленных значений не выполяется округления. - template - struct FunctionRoundingImpl::value>::type> + /** Реализация высокоуровневых функций округления для целочисленных значений. + */ + template + struct FunctionRoundingImpl::value && (scale_mode != NullScale)>::type> { + private: + using Op = IntegerRoundingComputation; + + public: static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { size_t size = in.size(); for (size_t i = 0; i < size; ++i) - out[i] = in[i]; + out[i] = Op::compute(in[i], scale); } static inline T apply(T val, size_t scale) { - return val; + return Op::compute(val, scale); } }; - template - struct FunctionRoundingImpl::value>::type> + /** Реализация высокоуровневых функций округления для значений с плавающей точкой. + */ + template + struct FunctionRoundingImpl::value && (scale_mode != NullScale)>::type> { private: - using Op = RoundingComputation; - using Data = typename Op::Data; + using Op = FloatRoundingComputation; + using Data = std::array; using Scale = typename Op::Scale; public: @@ -220,32 +376,22 @@ namespace DB Op::prepareScale(scale, mm_scale); const size_t size = in.size(); - const size_t data_size = std::tuple_size(); + const size_t data_count = std::tuple_size(); size_t i; - for (i = 0; i < (size - data_size + 1); i += data_size) - { - Data tmp; - for (size_t j = 0; j < data_size; ++j) - tmp[j] = in[i + j]; - - Data res; - Op::compute(tmp, mm_scale, res); - - for (size_t j = 0; j < data_size; ++j) - out[i + j] = res[j]; - } + for (i = 0; i < (size - data_count + 1); i += data_count) + Op::compute(reinterpret_cast(&in[i]), mm_scale, reinterpret_cast(&out[i])); if (i < size) { Data tmp{0}; - for (size_t j = 0; (j < data_size) && ((i + j) < size); ++j) + for (size_t j = 0; (j < data_count) && ((i + j) < size); ++j) tmp[j] = in[i + j]; Data res; - Op::compute(tmp, mm_scale, res); + Op::compute(reinterpret_cast(&tmp), mm_scale, reinterpret_cast(&res)); - for (size_t j = 0; (j < data_size) && ((i + j) < size); ++j) + for (size_t j = 0; (j < data_count) && ((i + j) < size); ++j) out[i + j] = res[j]; } } @@ -263,50 +409,29 @@ namespace DB tmp[0] = val; Data res; - Op::compute(tmp, mm_scale, res); + Op::compute(reinterpret_cast(&tmp), mm_scale, reinterpret_cast(&res)); return res[0]; } } }; - template - struct PrecisionForType + /** Реализация высокоуровневых функций округления в том случае, когда возвращается нулевое значение. + */ + template + struct FunctionRoundingImpl::type> { - template - static inline bool apply(const ColumnPtr & column, UInt8 & precision, - typename std::enable_if::value>::type * = nullptr) + public: + static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { - using ColumnType = ColumnConst; - - const ColumnType * precision_col = typeid_cast(&*column); - if (precision_col == nullptr) - return false; - - U val = precision_col->getData(); - if (val < 0) - val = 0; - else if (val >= static_cast(std::numeric_limits::digits10)) - val = static_cast(std::numeric_limits::digits10); - - precision = static_cast(val); - - return true; + size_t size = in.size(); + for (size_t i = 0; i < size; ++i) + out[i] = 0; } - /// Для целых чисел точность не имеет значения. - template - static inline bool apply(const ColumnPtr & column, UInt8 & precision, - typename std::enable_if::value>::type * = nullptr) + static inline T apply(T val, size_t scale) { - using ColumnType = ColumnConst; - - const ColumnType * precision_col = typeid_cast(&*column); - if (precision_col == nullptr) - return false; - - precision = 0; - - return true; + return 0; } }; @@ -360,10 +485,238 @@ namespace using result = typename FillArrayImpl::result; }; - /** Шаблон для функций, которые вычисляют приближенное значение входного параметра - * типа (U)Int8/16/32/64 или Float32/64 и принимают дополнительный необязятельный - * параметр указывающий сколько знаков после запятой оставить (по умолчанию - 0). - * Op - функция (round/floor/ceil) + /** Этот шаблон определяет точность, которую используют функции round/ceil/floor, + * затем преобразовывает её в значение, которое можно использовать в операциях + * умножения и деления. Поэтому оно называется масштабом. + */ + template + struct ScaleForRightType + { + }; + + template + struct ScaleForRightType::value + && std::is_signed::value>::type> + { + static inline bool apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + using PowersOf10 = typename FillArray::digits10 + 1>::result; + using ColumnType = ColumnConst; + + const ColumnType * precision_col = typeid_cast(&*column); + if (precision_col == nullptr) + return false; + + U val = precision_col->getData(); + if (val < 0) + { + if (val < -static_cast(std::numeric_limits::digits10)) + { + scale_mode = NullScale; + scale = 1; + } + else + { + scale_mode = NegativeScale; + scale = PowersOf10::values[-val]; + } + } + else if (val == 0) + { + scale_mode = ZeroScale; + scale = 1; + } + else + { + scale_mode = PositiveScale; + if (val > std::numeric_limits::digits10) + val = static_cast(std::numeric_limits::digits10); + scale = PowersOf10::values[val]; + } + + return true; + } + }; + + template + struct ScaleForRightType::value + && std::is_unsigned::value>::type> + { + static inline bool apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + using PowersOf10 = typename FillArray::digits10 + 1>::result; + using ColumnType = ColumnConst; + + const ColumnType * precision_col = typeid_cast(&*column); + if (precision_col == nullptr) + return false; + + U val = precision_col->getData(); + if (val == 0) + { + scale_mode = ZeroScale; + scale = 1; + } + else + { + scale_mode = PositiveScale; + if (val > static_cast(std::numeric_limits::digits10)) + val = static_cast(std::numeric_limits::digits10); + scale = PowersOf10::values[val]; + } + + return true; + } + }; + + template + struct ScaleForRightType::value + && std::is_signed::value>::type> + { + static inline bool apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + using PowersOf10 = typename FillArray::digits10 + 1>::result; + using ColumnType = ColumnConst; + + const ColumnType * precision_col = typeid_cast(&*column); + if (precision_col == nullptr) + return false; + + U val = precision_col->getData(); + if (val < 0) + { + if (val < -std::numeric_limits::digits10) + { + scale_mode = NullScale; + scale = 1; + } + else + { + scale_mode = NegativeScale; + scale = PowersOf10::values[-val]; + } + } + else + { + scale_mode = ZeroScale; + scale = 1; + } + + return true; + } + }; + + template + struct ScaleForRightType::value + && std::is_unsigned::value>::type> + { + static inline bool apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + using ColumnType = ColumnConst; + + const ColumnType * precision_col = typeid_cast(&*column); + if (precision_col == nullptr) + return false; + + scale_mode = ZeroScale; + scale = 1; + + return true; + } + }; + + /** Превратить параметр точности в масштаб. + */ + template + struct ScaleForLeftType + { + static inline void apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + if (!( ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale))) + { + throw Exception("Internal error", ErrorCodes::LOGICAL_ERROR); + } + } + }; + + /** Главный шаблон применяющий функцию округления к значению или столбцу. + */ + template + struct Cruncher + { + using Op = FunctionRoundingImpl; + + static inline void apply(Block & block, ColumnVector * col, const ColumnNumbers & arguments, size_t result, size_t scale) + { + ColumnVector * col_res = new ColumnVector; + block.getByPosition(result).column = col_res; + + typename ColumnVector::Container_t & vec_res = col_res->getData(); + vec_res.resize(col->getData().size()); + + Op::apply(col->getData(), scale, vec_res); + } + + static inline void apply(Block & block, ColumnConst * col, const ColumnNumbers & arguments, size_t result, size_t scale) + { + T res = Op::apply(col->getData(), scale); + ColumnConst * col_res = new ColumnConst(col->size(), res); + block.getByPosition(result).column = col_res; + } + }; + + /** Выбрать подходящий алгоритм обработки в зависимости от масштаба. + */ + template class U, int rounding_mode> + struct Dispatcher + { + static inline void apply(Block & block, U * col, const ColumnNumbers & arguments, size_t result) + { + ScaleMode scale_mode; + size_t scale; + + if (arguments.size() == 2) + ScaleForLeftType::apply(block.getByPosition(arguments[1]).column, scale_mode, scale); + else + { + scale_mode = ZeroScale; + scale = 1; + } + + if (scale_mode == PositiveScale) + Cruncher::apply(block, col, arguments, result, scale); + else if (scale_mode == ZeroScale) + Cruncher::apply(block, col, arguments, result, scale); + else if (scale_mode == NegativeScale) + Cruncher::apply(block, col, arguments, result, scale); + else if (scale_mode == NullScale) + Cruncher::apply(block, col, arguments, result, scale); + else + throw Exception("Illegal operation", ErrorCodes::LOGICAL_ERROR); + } + }; + + /** Шаблон для функций, которые округляют значение входного параметра типа + * (U)Int8/16/32/64 или Float32/64, и принимают дополнительный необязятельный + * параметр (по умолчанию - 0). */ template class FunctionRounding : public IFunction @@ -372,9 +725,6 @@ namespace static constexpr auto name = Name::name; static IFunction * create(const Context & context) { return new FunctionRounding; } - private: - using PowersOf10 = FillArray::digits10 + 1>::result; - private: template bool checkType(const IDataType * type) const @@ -385,72 +735,18 @@ namespace template bool executeForType(Block & block, const ColumnNumbers & arguments, size_t result) { - using OpWithScale = FunctionRoundingImpl; - using OpWithoutScale = FunctionRoundingImpl; - if (ColumnVector * col = typeid_cast *>(&*block.getByPosition(arguments[0]).column)) { - ColumnVector * col_res = new ColumnVector; - block.getByPosition(result).column = col_res; - - typename ColumnVector::Container_t & vec_res = col_res->getData(); - vec_res.resize(col->getData().size()); - - UInt8 precision = 0; - if (arguments.size() == 2) - precision = getPrecision(block.getByPosition(arguments[1]).column); - - if (precision > 0) - OpWithScale::apply(col->getData(), PowersOf10::values[precision], vec_res); - else - OpWithoutScale::apply(col->getData(), 0, vec_res); - + Dispatcher::apply(block, col, arguments, result); return true; } else if (ColumnConst * col = typeid_cast *>(&*block.getByPosition(arguments[0]).column)) { - UInt8 precision = 0; - if (arguments.size() == 2) - precision = getPrecision(block.getByPosition(arguments[1]).column); - - T res; - if (precision > 0) - res = OpWithScale::apply(col->getData(), PowersOf10::values[precision]); - else - res = OpWithoutScale::apply(col->getData(), 0); - - ColumnConst * col_res = new ColumnConst(col->size(), res); - block.getByPosition(result).column = col_res; - + Dispatcher::apply(block, col, arguments, result); return true; } - - return false; - } - - /// В зависимости от входного параметра, определить какая нужна точность - /// для результата. - template - UInt8 getPrecision(const ColumnPtr & column) - { - UInt8 precision = 0; - - if (!( PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision))) - { - throw Exception("Illegal column " + column->getName() - + " of second ('precision') argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - } - - return precision; + else + return false; } public: @@ -478,7 +774,9 @@ namespace || checkType(type) || checkType(type) || checkType(type) - || checkType(type))) + || checkType(type) + || checkType(type) + || checkType(type))) { throw Exception("Illegal type in second argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); From b7da487317c77c29c8f843823ae947a5221f0ee1 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 25 May 2015 16:37:08 +0300 Subject: [PATCH 015/109] dbms: Server: Removed useless include. [#METR-15210] --- dbms/include/DB/Functions/FunctionsRound.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index c01faab598f..bec76ea5005 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -5,8 +5,6 @@ #include #include -#include - namespace DB { From 9184d8781b3ef374dc58765c9c1142a3cc032dca Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 25 May 2015 16:38:06 +0300 Subject: [PATCH 016/109] dbms: Server: Cosmetic tweak. [#METR-15210] --- dbms/include/DB/Functions/FunctionsRound.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index bec76ea5005..b96e3002f05 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -5,6 +5,7 @@ #include #include + namespace DB { From 57287a28c12f8b41521be1353463112a825408ef Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 25 May 2015 16:39:06 +0300 Subject: [PATCH 017/109] Merge --- dbms/include/DB/Functions/FunctionsRound.h | 683 +++++++++++++++------ 1 file changed, 490 insertions(+), 193 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index ffe2769f24e..b96e3002f05 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -13,9 +13,14 @@ namespace DB * roundToExp2 - вниз до ближайшей степени двойки; * roundDuration - вниз до ближайшего из: 0, 1, 10, 30, 60, 120, 180, 240, 300, 600, 1200, 1800, 3600, 7200, 18000, 36000; * roundAge - вниз до ближайшего из: 0, 18, 25, 35, 45. - * round(x, N) - арифметическое округление (N - сколько знаков после запятой оставить; 0 по умолчанию). - * ceil(x, N) - наименьшее число, которое не меньше x (N - сколько знаков после запятой оставить; 0 по умолчанию). - * floor(x, N) - наибольшее число, которое не больше x (N - сколько знаков после запятой оставить; 0 по умолчанию). + * round(x, N) - арифметическое округление (N = 0 по умолчанию). + * ceil(x, N) - наименьшее число, которое не меньше x (N = 0 по умолчанию). + * floor(x, N) - наибольшее число, которое не больше x (N = 0 по умолчанию). + * + * Значение параметра N: + * - N > 0: округлять до числа с N десятичными знаками после запятой + * - N < 0: окурглять до целого числа с N нулевыми знаками + * - N = 0: округлять до целого числа */ template @@ -92,125 +97,275 @@ namespace DB } }; - /// Реализация функций округления на низком уровне. + /** Этот параметр контролирует поведение функций округления. + */ + enum ScaleMode + { + PositiveScale, // округлять до числа с N десятичными знаками после запятой + NegativeScale, // окурглять до целого числа с N нулевыми знаками + ZeroScale, // округлять до целого числа + NullScale // возвращать нулевое значение + }; - template - struct RoundingComputation + /** Реализация низкоуровневых функций округления для целочисленных значений. + */ + template + struct IntegerRoundingComputation { }; - template - struct RoundingComputation + template + struct IntegerRoundingComputation::value + && ((scale_mode == PositiveScale) || (scale_mode == ZeroScale))>::type> { - using Data = std::array; - using Scale = __m128; + static inline T compute(const T in, size_t scale) + { + return in; + } + }; - template - static inline void prepareScale(size_t scale, Scale & mm_scale, - typename std::enable_if::type * = nullptr) + template + struct IntegerRoundingComputation::value>::type> + { + static inline T compute(T in, size_t scale) + { + T rem = in % scale; + if (rem == in) + return 0; + + in -= rem; + if (static_cast(2 * rem) < scale) + return in; + else + return in + scale; + } + }; + + template + struct IntegerRoundingComputation::value>::type> + { + static inline T compute(const T in, size_t scale) + { + T rem = in % scale; + if (rem == in) + return 0; + + return in - rem + scale; + } + }; + + template + struct IntegerRoundingComputation::value>::type> + { + static inline T compute(const T in, size_t scale) + { + T rem = in % scale; + if (rem == in) + return 0; + + return in - rem; + } + }; + + template + struct BaseFloatRoundingComputation + { + }; + + template<> + struct BaseFloatRoundingComputation + { + using Scale = __m128; + static const size_t data_count = 4; + }; + + template<> + struct BaseFloatRoundingComputation + { + using Scale = __m128d; + static const size_t data_count = 2; + }; + + /** Реализация низкоуровневых функций округления для значений с плавающей точкой. + */ + template + struct FloatRoundingComputation : public BaseFloatRoundingComputation + { + }; + + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) { Float32 fscale = static_cast(scale); mm_scale = _mm_load1_ps(&fscale); } - template - static inline void prepareScale(size_t scale, Scale & mm_scale, - typename std::enable_if::type * = nullptr) + static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) { - } - - template - static inline void compute(const Data & in, const Scale & mm_scale, Data & out, - typename std::enable_if::type * = nullptr) - { - __m128 mm_value = _mm_loadu_ps(reinterpret_cast(&in)); - mm_value = _mm_mul_ps(mm_value, mm_scale); - mm_value = _mm_round_ps(mm_value, rounding_mode); - mm_value = _mm_div_ps(mm_value, mm_scale); - _mm_storeu_ps(reinterpret_cast(&out), mm_value); - } - - template - static inline void compute(const Data & in, const Scale & mm_scale, Data & out, - typename std::enable_if::type * = nullptr) - { - __m128 mm_value = _mm_loadu_ps(reinterpret_cast(&in)); - mm_value = _mm_round_ps(mm_value, rounding_mode); - _mm_storeu_ps(reinterpret_cast(&out), mm_value); + __m128 val = _mm_loadu_ps(in); + val = _mm_mul_ps(val, scale); + val = _mm_round_ps(val, rounding_mode); + val = _mm_div_ps(val, scale); + _mm_storeu_ps(out, val); } }; - template - struct RoundingComputation + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation { - using Data = std::array; - using Scale = __m128d; + static inline void prepareScale(size_t scale, Scale & mm_scale) + { + Float32 fscale = static_cast(scale); + mm_scale = _mm_load1_ps(&fscale); + } - template - static inline void prepareScale(size_t scale, Scale & mm_scale, - typename std::enable_if::type * = nullptr) + static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) + { + __m128 val = _mm_loadu_ps(in); + val = _mm_div_ps(val, scale); + __m128 res = _mm_cmpge_ps(val, getOne()); + val = _mm_round_ps(val, rounding_mode); + val = _mm_mul_ps(val, scale); + val = _mm_and_ps(val, res); + _mm_storeu_ps(out, val); + } + + private: + static inline const __m128 & getOne() + { + static const __m128 one = _mm_set1_ps(1.0); + return one; + } + }; + + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) + { + } + + static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) + { + __m128 val = _mm_loadu_ps(in); + val = _mm_round_ps(val, rounding_mode); + _mm_storeu_ps(out, val); + } + }; + + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) { Float64 fscale = static_cast(scale); mm_scale = _mm_load1_pd(&fscale); } - template - static inline void prepareScale(size_t scale, Scale & mm_scale, - typename std::enable_if::type * = nullptr) + static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) { - } - - template - static inline void compute(const Data & in, const Scale & mm_scale, Data & out, - typename std::enable_if::type * = nullptr) - { - __m128d mm_value = _mm_loadu_pd(reinterpret_cast(&in)); - mm_value = _mm_mul_pd(mm_value, mm_scale); - mm_value = _mm_round_pd(mm_value, rounding_mode); - mm_value = _mm_div_pd(mm_value, mm_scale); - _mm_storeu_pd(reinterpret_cast(&out), mm_value); - } - - template - static inline void compute(const Data & in, const Scale & mm_scale, Data & out, - typename std::enable_if::type * = nullptr) - { - __m128d mm_value = _mm_loadu_pd(reinterpret_cast(&in)); - mm_value = _mm_round_pd(mm_value, rounding_mode); - _mm_storeu_pd(reinterpret_cast(&out), mm_value); + __m128d val = _mm_loadu_pd(in); + val = _mm_mul_pd(val, scale); + val = _mm_round_pd(val, rounding_mode); + val = _mm_div_pd(val, scale); + _mm_storeu_pd(out, val); } }; - /// Реализация функций округления на высоком уровне. + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) + { + Float64 fscale = static_cast(scale); + mm_scale = _mm_load1_pd(&fscale); + } - template + static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) + { + __m128d val = _mm_loadu_pd(in); + val = _mm_div_pd(val, scale); + __m128d res = _mm_cmpge_pd(val, getOne()); + val = _mm_round_pd(val, rounding_mode); + val = _mm_mul_pd(val, scale); + val = _mm_and_pd(val, res); + _mm_storeu_pd(out, val); + } + + private: + static inline const __m128d & getOne() + { + static const __m128d one = _mm_set1_pd(1.0); + return one; + } + }; + + template + struct FloatRoundingComputation + : public BaseFloatRoundingComputation + { + static inline void prepareScale(size_t scale, Scale & mm_scale) + { + } + + static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) + { + __m128d val = _mm_loadu_pd(in); + val = _mm_round_pd(val, rounding_mode); + _mm_storeu_pd(out, val); + } + }; + + /** Реализация высокоуровневых функций округления. + */ + template struct FunctionRoundingImpl { }; - /// В случае целочисленных значений не выполяется округления. - template - struct FunctionRoundingImpl::value>::type> + /** Реализация высокоуровневых функций округления для целочисленных значений. + */ + template + struct FunctionRoundingImpl::value && (scale_mode != NullScale)>::type> { + private: + using Op = IntegerRoundingComputation; + + public: static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { size_t size = in.size(); for (size_t i = 0; i < size; ++i) - out[i] = in[i]; + out[i] = Op::compute(in[i], scale); } static inline T apply(T val, size_t scale) { - return val; + return Op::compute(val, scale); } }; - template - struct FunctionRoundingImpl::value>::type> + /** Реализация высокоуровневых функций округления для значений с плавающей точкой. + */ + template + struct FunctionRoundingImpl::value && (scale_mode != NullScale)>::type> { private: - using Op = RoundingComputation; - using Data = typename Op::Data; + using Op = FloatRoundingComputation; + using Data = std::array; using Scale = typename Op::Scale; public: @@ -220,32 +375,22 @@ namespace DB Op::prepareScale(scale, mm_scale); const size_t size = in.size(); - const size_t data_size = std::tuple_size(); + const size_t data_count = std::tuple_size(); size_t i; - for (i = 0; i < (size - data_size + 1); i += data_size) - { - Data tmp; - for (size_t j = 0; j < data_size; ++j) - tmp[j] = in[i + j]; - - Data res; - Op::compute(tmp, mm_scale, res); - - for (size_t j = 0; j < data_size; ++j) - out[i + j] = res[j]; - } + for (i = 0; i < (size - data_count + 1); i += data_count) + Op::compute(reinterpret_cast(&in[i]), mm_scale, reinterpret_cast(&out[i])); if (i < size) { Data tmp{0}; - for (size_t j = 0; (j < data_size) && ((i + j) < size); ++j) + for (size_t j = 0; (j < data_count) && ((i + j) < size); ++j) tmp[j] = in[i + j]; Data res; - Op::compute(tmp, mm_scale, res); + Op::compute(reinterpret_cast(&tmp), mm_scale, reinterpret_cast(&res)); - for (size_t j = 0; (j < data_size) && ((i + j) < size); ++j) + for (size_t j = 0; (j < data_count) && ((i + j) < size); ++j) out[i + j] = res[j]; } } @@ -263,50 +408,29 @@ namespace DB tmp[0] = val; Data res; - Op::compute(tmp, mm_scale, res); + Op::compute(reinterpret_cast(&tmp), mm_scale, reinterpret_cast(&res)); return res[0]; } } }; - template - struct PrecisionForType + /** Реализация высокоуровневых функций округления в том случае, когда возвращается нулевое значение. + */ + template + struct FunctionRoundingImpl::type> { - template - static inline bool apply(const ColumnPtr & column, UInt8 & precision, - typename std::enable_if::value>::type * = nullptr) + public: + static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { - using ColumnType = ColumnConst; - - const ColumnType * precision_col = typeid_cast(&*column); - if (precision_col == nullptr) - return false; - - U val = precision_col->getData(); - if (val < 0) - val = 0; - else if (val >= static_cast(std::numeric_limits::digits10)) - val = static_cast(std::numeric_limits::digits10); - - precision = static_cast(val); - - return true; + size_t size = in.size(); + for (size_t i = 0; i < size; ++i) + out[i] = 0; } - /// Для целых чисел точность не имеет значения. - template - static inline bool apply(const ColumnPtr & column, UInt8 & precision, - typename std::enable_if::value>::type * = nullptr) + static inline T apply(T val, size_t scale) { - using ColumnType = ColumnConst; - - const ColumnType * precision_col = typeid_cast(&*column); - if (precision_col == nullptr) - return false; - - precision = 0; - - return true; + return 0; } }; @@ -360,10 +484,238 @@ namespace using result = typename FillArrayImpl::result; }; - /** Шаблон для функций, которые вычисляют приближенное значение входного параметра - * типа (U)Int8/16/32/64 или Float32/64 и принимают дополнительный необязятельный - * параметр указывающий сколько знаков после запятой оставить (по умолчанию - 0). - * Op - функция (round/floor/ceil) + /** Этот шаблон определяет точность, которую используют функции round/ceil/floor, + * затем преобразовывает её в значение, которое можно использовать в операциях + * умножения и деления. Поэтому оно называется масштабом. + */ + template + struct ScaleForRightType + { + }; + + template + struct ScaleForRightType::value + && std::is_signed::value>::type> + { + static inline bool apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + using PowersOf10 = typename FillArray::digits10 + 1>::result; + using ColumnType = ColumnConst; + + const ColumnType * precision_col = typeid_cast(&*column); + if (precision_col == nullptr) + return false; + + U val = precision_col->getData(); + if (val < 0) + { + if (val < -static_cast(std::numeric_limits::digits10)) + { + scale_mode = NullScale; + scale = 1; + } + else + { + scale_mode = NegativeScale; + scale = PowersOf10::values[-val]; + } + } + else if (val == 0) + { + scale_mode = ZeroScale; + scale = 1; + } + else + { + scale_mode = PositiveScale; + if (val > std::numeric_limits::digits10) + val = static_cast(std::numeric_limits::digits10); + scale = PowersOf10::values[val]; + } + + return true; + } + }; + + template + struct ScaleForRightType::value + && std::is_unsigned::value>::type> + { + static inline bool apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + using PowersOf10 = typename FillArray::digits10 + 1>::result; + using ColumnType = ColumnConst; + + const ColumnType * precision_col = typeid_cast(&*column); + if (precision_col == nullptr) + return false; + + U val = precision_col->getData(); + if (val == 0) + { + scale_mode = ZeroScale; + scale = 1; + } + else + { + scale_mode = PositiveScale; + if (val > static_cast(std::numeric_limits::digits10)) + val = static_cast(std::numeric_limits::digits10); + scale = PowersOf10::values[val]; + } + + return true; + } + }; + + template + struct ScaleForRightType::value + && std::is_signed::value>::type> + { + static inline bool apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + using PowersOf10 = typename FillArray::digits10 + 1>::result; + using ColumnType = ColumnConst; + + const ColumnType * precision_col = typeid_cast(&*column); + if (precision_col == nullptr) + return false; + + U val = precision_col->getData(); + if (val < 0) + { + if (val < -std::numeric_limits::digits10) + { + scale_mode = NullScale; + scale = 1; + } + else + { + scale_mode = NegativeScale; + scale = PowersOf10::values[-val]; + } + } + else + { + scale_mode = ZeroScale; + scale = 1; + } + + return true; + } + }; + + template + struct ScaleForRightType::value + && std::is_unsigned::value>::type> + { + static inline bool apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + using ColumnType = ColumnConst; + + const ColumnType * precision_col = typeid_cast(&*column); + if (precision_col == nullptr) + return false; + + scale_mode = ZeroScale; + scale = 1; + + return true; + } + }; + + /** Превратить параметр точности в масштаб. + */ + template + struct ScaleForLeftType + { + static inline void apply(const ColumnPtr & column, ScaleMode & scale_mode, size_t & scale) + { + if (!( ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale) + || ScaleForRightType::apply(column, scale_mode, scale))) + { + throw Exception("Internal error", ErrorCodes::LOGICAL_ERROR); + } + } + }; + + /** Главный шаблон применяющий функцию округления к значению или столбцу. + */ + template + struct Cruncher + { + using Op = FunctionRoundingImpl; + + static inline void apply(Block & block, ColumnVector * col, const ColumnNumbers & arguments, size_t result, size_t scale) + { + ColumnVector * col_res = new ColumnVector; + block.getByPosition(result).column = col_res; + + typename ColumnVector::Container_t & vec_res = col_res->getData(); + vec_res.resize(col->getData().size()); + + Op::apply(col->getData(), scale, vec_res); + } + + static inline void apply(Block & block, ColumnConst * col, const ColumnNumbers & arguments, size_t result, size_t scale) + { + T res = Op::apply(col->getData(), scale); + ColumnConst * col_res = new ColumnConst(col->size(), res); + block.getByPosition(result).column = col_res; + } + }; + + /** Выбрать подходящий алгоритм обработки в зависимости от масштаба. + */ + template class U, int rounding_mode> + struct Dispatcher + { + static inline void apply(Block & block, U * col, const ColumnNumbers & arguments, size_t result) + { + ScaleMode scale_mode; + size_t scale; + + if (arguments.size() == 2) + ScaleForLeftType::apply(block.getByPosition(arguments[1]).column, scale_mode, scale); + else + { + scale_mode = ZeroScale; + scale = 1; + } + + if (scale_mode == PositiveScale) + Cruncher::apply(block, col, arguments, result, scale); + else if (scale_mode == ZeroScale) + Cruncher::apply(block, col, arguments, result, scale); + else if (scale_mode == NegativeScale) + Cruncher::apply(block, col, arguments, result, scale); + else if (scale_mode == NullScale) + Cruncher::apply(block, col, arguments, result, scale); + else + throw Exception("Illegal operation", ErrorCodes::LOGICAL_ERROR); + } + }; + + /** Шаблон для функций, которые округляют значение входного параметра типа + * (U)Int8/16/32/64 или Float32/64, и принимают дополнительный необязятельный + * параметр (по умолчанию - 0). */ template class FunctionRounding : public IFunction @@ -372,9 +724,6 @@ namespace static constexpr auto name = Name::name; static IFunction * create(const Context & context) { return new FunctionRounding; } - private: - using PowersOf10 = FillArray::digits10 + 1>::result; - private: template bool checkType(const IDataType * type) const @@ -385,72 +734,18 @@ namespace template bool executeForType(Block & block, const ColumnNumbers & arguments, size_t result) { - using OpWithScale = FunctionRoundingImpl; - using OpWithoutScale = FunctionRoundingImpl; - if (ColumnVector * col = typeid_cast *>(&*block.getByPosition(arguments[0]).column)) { - ColumnVector * col_res = new ColumnVector; - block.getByPosition(result).column = col_res; - - typename ColumnVector::Container_t & vec_res = col_res->getData(); - vec_res.resize(col->getData().size()); - - UInt8 precision = 0; - if (arguments.size() == 2) - precision = getPrecision(block.getByPosition(arguments[1]).column); - - if (precision > 0) - OpWithScale::apply(col->getData(), PowersOf10::values[precision], vec_res); - else - OpWithoutScale::apply(col->getData(), 0, vec_res); - + Dispatcher::apply(block, col, arguments, result); return true; } else if (ColumnConst * col = typeid_cast *>(&*block.getByPosition(arguments[0]).column)) { - UInt8 precision = 0; - if (arguments.size() == 2) - precision = getPrecision(block.getByPosition(arguments[1]).column); - - T res; - if (precision > 0) - res = OpWithScale::apply(col->getData(), PowersOf10::values[precision]); - else - res = OpWithoutScale::apply(col->getData(), 0); - - ColumnConst * col_res = new ColumnConst(col->size(), res); - block.getByPosition(result).column = col_res; - + Dispatcher::apply(block, col, arguments, result); return true; } - - return false; - } - - /// В зависимости от входного параметра, определить какая нужна точность - /// для результата. - template - UInt8 getPrecision(const ColumnPtr & column) - { - UInt8 precision = 0; - - if (!( PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision) - || PrecisionForType::apply(column, precision))) - { - throw Exception("Illegal column " + column->getName() - + " of second ('precision') argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - } - - return precision; + else + return false; } public: @@ -478,7 +773,9 @@ namespace || checkType(type) || checkType(type) || checkType(type) - || checkType(type))) + || checkType(type) + || checkType(type) + || checkType(type))) { throw Exception("Illegal type in second argument of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); From 1143c9def94a3c8b8a0945a354d79d482452494d Mon Sep 17 00:00:00 2001 From: Sergey Magidovich Date: Mon, 25 May 2015 17:10:30 +0300 Subject: [PATCH 018/109] statdaemons: Add test. [#MOBMET-1661] --- dbms/include/DB/Common/SipHash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/include/DB/Common/SipHash.h b/dbms/include/DB/Common/SipHash.h index bcfe81b93ed..2892929fc09 100644 --- a/dbms/include/DB/Common/SipHash.h +++ b/dbms/include/DB/Common/SipHash.h @@ -12,7 +12,7 @@ */ #include - +#include #define ROTL(x,b) (u64)( ((x) << (b)) | ( (x) >> (64 - (b))) ) From ab3befbf9221ea3b7a32ee17343d5d090ae7ae81 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 25 May 2015 18:04:21 +0300 Subject: [PATCH 019/109] dbms: Server: Fixed MySQL-conformance issue after testing. [#METR-15210] --- dbms/include/DB/Functions/FunctionsRound.h | 29 +++++++--------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index b96e3002f05..dc27ad45c32 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -132,9 +132,6 @@ namespace DB static inline T compute(T in, size_t scale) { T rem = in % scale; - if (rem == in) - return 0; - in -= rem; if (static_cast(2 * rem) < scale) return in; @@ -150,9 +147,6 @@ namespace DB static inline T compute(const T in, size_t scale) { T rem = in % scale; - if (rem == in) - return 0; - return in - rem + scale; } }; @@ -164,9 +158,6 @@ namespace DB static inline T compute(const T in, size_t scale) { T rem = in % scale; - if (rem == in) - return 0; - return in - rem; } }; @@ -231,7 +222,7 @@ namespace DB { __m128 val = _mm_loadu_ps(in); val = _mm_div_ps(val, scale); - __m128 res = _mm_cmpge_ps(val, getOne()); + __m128 res = _mm_cmpge_ps(val, getOneTenth()); val = _mm_round_ps(val, rounding_mode); val = _mm_mul_ps(val, scale); val = _mm_and_ps(val, res); @@ -239,10 +230,10 @@ namespace DB } private: - static inline const __m128 & getOne() + static inline const __m128 & getOneTenth() { - static const __m128 one = _mm_set1_ps(1.0); - return one; + static const __m128 one_tenth = _mm_set1_ps(0.1); + return one_tenth; } }; @@ -296,7 +287,7 @@ namespace DB { __m128d val = _mm_loadu_pd(in); val = _mm_div_pd(val, scale); - __m128d res = _mm_cmpge_pd(val, getOne()); + __m128d res = _mm_cmpge_pd(val, getOneTenth()); val = _mm_round_pd(val, rounding_mode); val = _mm_mul_pd(val, scale); val = _mm_and_pd(val, res); @@ -304,10 +295,10 @@ namespace DB } private: - static inline const __m128d & getOne() + static inline const __m128d & getOneTenth() { - static const __m128d one = _mm_set1_pd(1.0); - return one; + static const __m128d one_tenth = _mm_set1_pd(0.1); + return one_tenth; } }; @@ -423,9 +414,7 @@ namespace DB public: static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { - size_t size = in.size(); - for (size_t i = 0; i < size; ++i) - out[i] = 0; + ::memset(reinterpret_cast(&out[0]), 0, in.size()); } static inline T apply(T val, size_t scale) From c965ef892a753da4f6ac65c3d7bb76db8fc0bdc1 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 25 May 2015 18:09:12 +0300 Subject: [PATCH 020/109] Merge --- dbms/include/DB/Functions/FunctionsRound.h | 29 +++++++--------------- 1 file changed, 9 insertions(+), 20 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index b96e3002f05..dc27ad45c32 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -132,9 +132,6 @@ namespace DB static inline T compute(T in, size_t scale) { T rem = in % scale; - if (rem == in) - return 0; - in -= rem; if (static_cast(2 * rem) < scale) return in; @@ -150,9 +147,6 @@ namespace DB static inline T compute(const T in, size_t scale) { T rem = in % scale; - if (rem == in) - return 0; - return in - rem + scale; } }; @@ -164,9 +158,6 @@ namespace DB static inline T compute(const T in, size_t scale) { T rem = in % scale; - if (rem == in) - return 0; - return in - rem; } }; @@ -231,7 +222,7 @@ namespace DB { __m128 val = _mm_loadu_ps(in); val = _mm_div_ps(val, scale); - __m128 res = _mm_cmpge_ps(val, getOne()); + __m128 res = _mm_cmpge_ps(val, getOneTenth()); val = _mm_round_ps(val, rounding_mode); val = _mm_mul_ps(val, scale); val = _mm_and_ps(val, res); @@ -239,10 +230,10 @@ namespace DB } private: - static inline const __m128 & getOne() + static inline const __m128 & getOneTenth() { - static const __m128 one = _mm_set1_ps(1.0); - return one; + static const __m128 one_tenth = _mm_set1_ps(0.1); + return one_tenth; } }; @@ -296,7 +287,7 @@ namespace DB { __m128d val = _mm_loadu_pd(in); val = _mm_div_pd(val, scale); - __m128d res = _mm_cmpge_pd(val, getOne()); + __m128d res = _mm_cmpge_pd(val, getOneTenth()); val = _mm_round_pd(val, rounding_mode); val = _mm_mul_pd(val, scale); val = _mm_and_pd(val, res); @@ -304,10 +295,10 @@ namespace DB } private: - static inline const __m128d & getOne() + static inline const __m128d & getOneTenth() { - static const __m128d one = _mm_set1_pd(1.0); - return one; + static const __m128d one_tenth = _mm_set1_pd(0.1); + return one_tenth; } }; @@ -423,9 +414,7 @@ namespace DB public: static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { - size_t size = in.size(); - for (size_t i = 0; i < size; ++i) - out[i] = 0; + ::memset(reinterpret_cast(&out[0]), 0, in.size()); } static inline T apply(T val, size_t scale) From fef078c8ecc2abbedc4e4ef83bba701205bc72a0 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Mon, 25 May 2015 18:33:34 +0300 Subject: [PATCH 021/109] Merge --- .../SummingSortedBlockInputStream.h | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h b/dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h index 1ded9c19d2b..c05bc4dc340 100644 --- a/dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h +++ b/dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h @@ -128,8 +128,10 @@ private: * все элементы - нулевые. */ template - void mergeMaps(Row & row, TSortCursor & cursor) + bool mergeMaps(Row & row, TSortCursor & cursor) { + auto non_empty_map_present = false; + /// merge nested maps for (const auto & map : maps_to_sum) { @@ -235,11 +237,26 @@ private: else break; + /// discard last row if necessary + if (discard_prev) + key_array_result.pop_back(); + /// store results into accumulator-row key_array_lhs = std::move(key_array_result); for (const auto val_col_index : ext::range(0, val_count)) + { + /// discard last row if necessary + if (discard_prev) + val_arrays_result[val_col_index].pop_back(); + row[map.val_col_nums[val_col_index]].get() = std::move(val_arrays_result[val_col_index]); + } + + if (!key_array_lhs.empty()) + non_empty_map_present = true; } + + return non_empty_map_present; } /** Прибавить строчку под курсором к row. @@ -248,9 +265,7 @@ private: template bool addRow(Row & row, TSortCursor & cursor) { - mergeMaps(row, cursor); - - bool res = false; /// Есть ли хотя бы одно ненулевое число. + bool res = mergeMaps(row, cursor); /// Есть ли хотя бы одно ненулевое число или непустой массив for (size_t i = 0, size = column_numbers_to_sum.size(); i < size; ++i) { From d34ed382e0e34f3e01d69bee8662ebc389243ef2 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 25 May 2015 19:13:10 +0300 Subject: [PATCH 022/109] dbms: Server: Added functional test for rounding functions. [#METR-15210] --- .../00161_rounding_functions.reference | 799 +++++++++++++++ .../0_stateless/00161_rounding_functions.sql | 940 ++++++++++++++++++ 2 files changed, 1739 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00161_rounding_functions.reference create mode 100644 dbms/tests/queries/0_stateless/00161_rounding_functions.sql diff --git a/dbms/tests/queries/0_stateless/00161_rounding_functions.reference b/dbms/tests/queries/0_stateless/00161_rounding_functions.reference new file mode 100644 index 00000000000..f924a9f5d0d --- /dev/null +++ b/dbms/tests/queries/0_stateless/00161_rounding_functions.reference @@ -0,0 +1,799 @@ +0 0 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +3 3 2 +2 2 2 +-3 -2 -3 +-2 -2 -3 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 0 0 +0 0 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 0 0 +0 0 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +13 13 13 +10 20 10 +10 20 10 +10 20 10 +10 20 10 +13 20 10 +13 20 10 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +0 100 0 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-16 -6 -16 +-16 -6 -16 +-16 -6 -16 +-16 -6 -16 +-13 -6 -16 +-13 -6 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +0 0 0 +0 0 0 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-16 -6 -16 +-16 -6 -16 +-16 -6 -16 +-16 -6 -16 +-13 -6 -16 +-13 -6 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-16 -6 -16 +-16 -6 -16 +-16 -6 -16 +-16 -6 -16 +-13 -6 -16 +-13 -6 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-16 -6 -16 +-16 -6 -16 +-16 -6 -16 +-16 -6 -16 +-13 -6 -16 +-13 -6 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-16 84 -16 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +-13 0 0 +-13 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +-13 -13 -13 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +-13 0 0 +-13 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.72 2.72 2.71 +2.7 2.8 2.7 +2.7 2.8 2.7 +2.7 2.8 2.7 +2.7 2.8 2.7 +2.7 2.8 2.7 +2.7 2.8 2.7 +2.7 2.8 2.7 +2.7 2.8 2.7 +2.7 2.8 2.7 +2.7 2.8 2.7 +3 3 2 +3 3 2 +3 3 2 +3 3 2 +3 3 2 +3 3 2 +3 3 2 +3 3 2 +3 3 2 +0 10 0 +0 10 0 +0 10 0 +0 10 0 +2.7 10 0 +2.7 10 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.72 -2.71 -2.72 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-2.7 -2.7 -2.8 +-3 -2 -3 +-3 -2 -3 +-3 -2 -3 +-3 -2 -3 +-3 -2 -3 +-3 -2 -3 +-3 -2 -3 +-3 -2 -3 +-3 -2 -3 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +-2.7 0 0 +-2.7 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +13112220 13112230 13112220 +13112200 13112300 13112200 +13112000 13113000 13112000 +13110000 13120000 13110000 +13100000 13200000 13100000 +13000000 14000000 13000000 +10000000 20000000 10000000 +0 100000000 0 +0 1000000000 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +2.7 2.8 2.7 +2.72 2.72 2.71 +2.718 2.719 2.718 +2.7183 2.7183 2.7182 +2.71828 2.71829 2.71828 +2.718282 2.718282 2.718281 +2.7182818 2.7182819 2.7182818 +2.71828183 2.71828183 2.71828182 +2.718281828 2.718281829 2.718281828 +2.7182818285 2.7182818285 2.7182818284 +2.71828182846 2.71828182846 2.71828182845 +2.718281828459 2.71828182846 2.718281828459 +2.718281828459 2.7182818284591 2.718281828459 +2.71828182845904 2.71828182845905 2.71828182845904 +2.718281828459045 2.718281828459045 2.718281828459045 +2.718281828459045 2.718281828459045 2.718281828459045 +2.718281828459045 2.718281828459045 2.718281828459045 +2.718281828459045 2.718281828459045 2.718281828459045 +2.718281828459045 2.718281828459045 2.718281828459045 +2.718281828459045 2.718281828459045 2.718281828459045 diff --git a/dbms/tests/queries/0_stateless/00161_rounding_functions.sql b/dbms/tests/queries/0_stateless/00161_rounding_functions.sql new file mode 100644 index 00000000000..7851f8e02c2 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00161_rounding_functions.sql @@ -0,0 +1,940 @@ +/* Без дополнительного параметра */ + +SELECT round(0), ceil(0), floor(0); + +SELECT round(toUInt8(13)), ceil(toUInt8(13)), floor(toUInt8(13)); +SELECT round(toUInt16(13)), ceil(toUInt16(13)), floor(toUInt16(13)); +SELECT round(toUInt32(13)), ceil(toUInt32(13)), floor(toUInt32(13)); +SELECT round(toUInt64(13)), ceil(toUInt64(13)), floor(toUInt64(13)); +SELECT round(toInt8(13)), ceil(toInt8(13)), floor(toInt8(13)); +SELECT round(toInt16(13)), ceil(toInt16(13)), floor(toInt16(13)); +SELECT round(toInt32(13)), ceil(toInt32(13)), floor(toInt32(13)); +SELECT round(toInt64(13)), ceil(toInt64(13)), floor(toInt64(13)); +SELECT round(toFloat32(13)), ceil(toFloat32(13)), floor(toFloat32(13)); +SELECT round(toFloat64(13)), ceil(toFloat64(13)), floor(toFloat64(13)); + +SELECT round(toInt8(-13)), ceil(toInt8(-13)), floor(toInt8(-13)); +SELECT round(toInt16(-13)), ceil(toInt16(-13)), floor(toInt16(-13)); +SELECT round(toInt32(-13)), ceil(toInt32(-13)), floor(toInt32(-13)); +SELECT round(toInt64(-13)), ceil(toInt64(-13)), floor(toInt64(-13)); +SELECT round(toFloat32(-13)), ceil(toFloat32(-13)), floor(toFloat32(-13)); +SELECT round(toFloat64(-13)), ceil(toFloat64(-13)), floor(toFloat64(-13)); + +SELECT round(2.7), ceil(2.7), floor(2.7); +SELECT round(2.1), ceil(2,1), floor(2.1); + +SELECT round(-2.7), ceil(-2.7), floor(-2.7); +SELECT round(-2.1), ceil(-2,1), floor(-2.1); + +/* UInt8 */ + +SELECT round(toUInt8(13), toUInt8(2)), ceil(toUInt8(13), toUInt8(2)), floor(toUInt8(13), toUInt8(2)); +SELECT round(toUInt8(13), toUInt16(2)), ceil(toUInt8(13), toUInt16(2)), floor(toUInt8(13), toUInt16(2)); +SELECT round(toUInt8(13), toUInt32(2)), ceil(toUInt8(13), toUInt32(2)), floor(toUInt8(13), toUInt32(2)); +SELECT round(toUInt8(13), toUInt64(2)), ceil(toUInt8(13), toUInt64(2)), floor(toUInt8(13), toUInt64(2)); +SELECT round(toUInt8(13), toInt8(2)), ceil(toUInt8(13), toInt8(2)), floor(toUInt8(13), toInt8(2)); +SELECT round(toUInt8(13), toInt16(2)), ceil(toUInt8(13), toInt16(2)), floor(toUInt8(13), toInt16(2)); +SELECT round(toUInt8(13), toInt32(2)), ceil(toUInt8(13), toInt32(2)), floor(toUInt8(13), toInt32(2)); +SELECT round(toUInt8(13), toInt64(2)), ceil(toUInt8(13), toInt64(2)), floor(toUInt8(13), toInt64(2)); +SELECT round(toUInt8(13), toFloat32(2.1)), ceil(toUInt8(13), toFloat32(2.1)), floor(toUInt8(13), toFloat32(2.1)); +SELECT round(toUInt8(13), toFloat64(2.1)), ceil(toUInt8(13), toFloat64(2.1)), floor(toUInt8(13), toFloat64(2.1)); + +SELECT round(toUInt8(13), toUInt8(1)), ceil(toUInt8(13), toUInt8(1)), floor(toUInt8(13), toUInt8(1)); +SELECT round(toUInt8(13), toUInt16(1)), ceil(toUInt8(13), toUInt16(1)), floor(toUInt8(13), toUInt16(1)); +SELECT round(toUInt8(13), toUInt32(1)), ceil(toUInt8(13), toUInt32(1)), floor(toUInt8(13), toUInt32(1)); +SELECT round(toUInt8(13), toUInt64(1)), ceil(toUInt8(13), toUInt64(1)), floor(toUInt8(13), toUInt64(1)); +SELECT round(toUInt8(13), toInt8(1)), ceil(toUInt8(13), toInt8(1)), floor(toUInt8(13), toInt8(1)); +SELECT round(toUInt8(13), toInt16(1)), ceil(toUInt8(13), toInt16(1)), floor(toUInt8(13), toInt16(1)); +SELECT round(toUInt8(13), toInt32(1)), ceil(toUInt8(13), toInt32(1)), floor(toUInt8(13), toInt32(1)); +SELECT round(toUInt8(13), toInt64(1)), ceil(toUInt8(13), toInt64(1)), floor(toUInt8(13), toInt64(1)); +SELECT round(toUInt8(13), toFloat32(1.1)), ceil(toUInt8(13), toFloat32(1.1)), floor(toUInt8(13), toFloat32(1.1)); +SELECT round(toUInt8(13), toFloat64(1.1)), ceil(toUInt8(13), toFloat64(1.1)), floor(toUInt8(13), toFloat64(1.1)); + +SELECT round(toUInt8(13), toUInt16(0)), ceil(toUInt8(13), toUInt16(0)), floor(toUInt8(13), toUInt16(0)); +SELECT round(toUInt8(13), toUInt32(0)), ceil(toUInt8(13), toUInt32(0)), floor(toUInt8(13), toUInt32(0)); +SELECT round(toUInt8(13), toUInt64(0)), ceil(toUInt8(13), toUInt64(0)), floor(toUInt8(13), toUInt64(0)); +SELECT round(toUInt8(13), toInt8(0)), ceil(toUInt8(13), toInt8(0)), floor(toUInt8(13), toInt8(0)); +SELECT round(toUInt8(13), toInt16(0)), ceil(toUInt8(13), toInt16(0)), floor(toUInt8(13), toInt16(0)); +SELECT round(toUInt8(13), toInt32(0)), ceil(toUInt8(13), toInt32(0)), floor(toUInt8(13), toInt32(0)); +SELECT round(toUInt8(13), toInt64(0)), ceil(toUInt8(13), toInt64(0)), floor(toUInt8(13), toInt64(0)); +SELECT round(toUInt8(13), toFloat32(0.1)), ceil(toUInt8(13), toFloat32(0.1)), floor(toUInt8(13), toFloat32(0.1)); +SELECT round(toUInt8(13), toFloat64(0.1)), ceil(toUInt8(13), toFloat64(0.1)), floor(toUInt8(13), toFloat64(0.1)); + +SELECT round(toUInt8(13), toInt8(-1)), ceil(toUInt8(13), toInt8(-1)), floor(toUInt8(13), toInt8(-1)); +SELECT round(toUInt8(13), toInt16(-1)), ceil(toUInt8(13), toInt16(-1)), floor(toUInt8(13), toInt16(-1)); +SELECT round(toUInt8(13), toInt32(-1)), ceil(toUInt8(13), toInt32(-1)), floor(toUInt8(13), toInt32(-1)); +SELECT round(toUInt8(13), toInt64(-1)), ceil(toUInt8(13), toInt64(-1)), floor(toUInt8(13), toInt64(-1)); +SELECT round(toUInt8(13), toFloat32(1.1)), ceil(toUInt8(13), toFloat32(-1.1)), floor(toUInt8(13), toFloat32(-1.1)); +SELECT round(toUInt8(13), toFloat64(1.1)), ceil(toUInt8(13), toFloat64(-1.1)), floor(toUInt8(13), toFloat64(-1.1)); + +SELECT round(toUInt8(13), toInt8(-2)), ceil(toUInt8(13), toInt8(-2)), floor(toUInt8(13), toInt8(-2)); +SELECT round(toUInt8(13), toInt16(-2)), ceil(toUInt8(13), toInt16(-2)), floor(toUInt8(13), toInt16(-2)); +SELECT round(toUInt8(13), toInt32(-2)), ceil(toUInt8(13), toInt32(-2)), floor(toUInt8(13), toInt32(-2)); +SELECT round(toUInt8(13), toInt64(-2)), ceil(toUInt8(13), toInt64(-2)), floor(toUInt8(13), toInt64(-2)); +SELECT round(toUInt8(13), toFloat32(-2.1)), ceil(toUInt8(13), toFloat32(-2.1)), floor(toUInt8(13), toFloat32(-2.1)); +SELECT round(toUInt8(13), toFloat64(-2.1)), ceil(toUInt8(13), toFloat64(-2.1)), floor(toUInt8(13), toFloat64(-2.1)); + +/* UInt16 */ + +SELECT round(toUInt16(13), toUInt8(2)), ceil(toUInt16(13), toUInt8(2)), floor(toUInt16(13), toUInt8(2)); +SELECT round(toUInt16(13), toUInt16(2)), ceil(toUInt16(13), toUInt16(2)), floor(toUInt16(13), toUInt16(2)); +SELECT round(toUInt16(13), toUInt32(2)), ceil(toUInt16(13), toUInt32(2)), floor(toUInt16(13), toUInt32(2)); +SELECT round(toUInt16(13), toUInt64(2)), ceil(toUInt16(13), toUInt64(2)), floor(toUInt16(13), toUInt64(2)); +SELECT round(toUInt16(13), toInt8(2)), ceil(toUInt16(13), toInt8(2)), floor(toUInt16(13), toInt8(2)); +SELECT round(toUInt16(13), toInt16(2)), ceil(toUInt16(13), toInt16(2)), floor(toUInt16(13), toInt16(2)); +SELECT round(toUInt16(13), toInt32(2)), ceil(toUInt16(13), toInt32(2)), floor(toUInt16(13), toInt32(2)); +SELECT round(toUInt16(13), toInt64(2)), ceil(toUInt16(13), toInt64(2)), floor(toUInt16(13), toInt64(2)); +SELECT round(toUInt16(13), toFloat32(2.1)), ceil(toUInt16(13), toFloat32(2.1)), floor(toUInt16(13), toFloat32(2.1)); +SELECT round(toUInt16(13), toFloat64(2.1)), ceil(toUInt16(13), toFloat64(2.1)), floor(toUInt16(13), toFloat64(2.1)); + +SELECT round(toUInt16(13), toUInt8(1)), ceil(toUInt16(13), toUInt8(1)), floor(toUInt16(13), toUInt8(1)); +SELECT round(toUInt16(13), toUInt16(1)), ceil(toUInt16(13), toUInt16(1)), floor(toUInt16(13), toUInt16(1)); +SELECT round(toUInt16(13), toUInt32(1)), ceil(toUInt16(13), toUInt32(1)), floor(toUInt16(13), toUInt32(1)); +SELECT round(toUInt16(13), toUInt64(1)), ceil(toUInt16(13), toUInt64(1)), floor(toUInt16(13), toUInt64(1)); +SELECT round(toUInt16(13), toInt8(1)), ceil(toUInt16(13), toInt8(1)), floor(toUInt16(13), toInt8(1)); +SELECT round(toUInt16(13), toInt16(1)), ceil(toUInt16(13), toInt16(1)), floor(toUInt16(13), toInt16(1)); +SELECT round(toUInt16(13), toInt32(1)), ceil(toUInt16(13), toInt32(1)), floor(toUInt16(13), toInt32(1)); +SELECT round(toUInt16(13), toInt64(1)), ceil(toUInt16(13), toInt64(1)), floor(toUInt16(13), toInt64(1)); +SELECT round(toUInt16(13), toFloat32(1.1)), ceil(toUInt16(13), toFloat32(1.1)), floor(toUInt16(13), toFloat32(1.1)); +SELECT round(toUInt16(13), toFloat64(1.1)), ceil(toUInt16(13), toFloat64(1.1)), floor(toUInt16(13), toFloat64(1.1)); + +SELECT round(toUInt16(13), toUInt16(0)), ceil(toUInt16(13), toUInt16(0)), floor(toUInt16(13), toUInt16(0)); +SELECT round(toUInt16(13), toUInt32(0)), ceil(toUInt16(13), toUInt32(0)), floor(toUInt16(13), toUInt32(0)); +SELECT round(toUInt16(13), toUInt64(0)), ceil(toUInt16(13), toUInt64(0)), floor(toUInt16(13), toUInt64(0)); +SELECT round(toUInt16(13), toInt8(0)), ceil(toUInt16(13), toInt8(0)), floor(toUInt16(13), toInt8(0)); +SELECT round(toUInt16(13), toInt16(0)), ceil(toUInt16(13), toInt16(0)), floor(toUInt16(13), toInt16(0)); +SELECT round(toUInt16(13), toInt32(0)), ceil(toUInt16(13), toInt32(0)), floor(toUInt16(13), toInt32(0)); +SELECT round(toUInt16(13), toInt64(0)), ceil(toUInt16(13), toInt64(0)), floor(toUInt16(13), toInt64(0)); +SELECT round(toUInt16(13), toFloat32(0.1)), ceil(toUInt16(13), toFloat32(0.1)), floor(toUInt16(13), toFloat32(0.1)); +SELECT round(toUInt16(13), toFloat64(0.1)), ceil(toUInt16(13), toFloat64(0.1)), floor(toUInt16(13), toFloat64(0.1)); + +SELECT round(toUInt16(13), toInt8(-1)), ceil(toUInt16(13), toInt8(-1)), floor(toUInt16(13), toInt8(-1)); +SELECT round(toUInt16(13), toInt16(-1)), ceil(toUInt16(13), toInt16(-1)), floor(toUInt16(13), toInt16(-1)); +SELECT round(toUInt16(13), toInt32(-1)), ceil(toUInt16(13), toInt32(-1)), floor(toUInt16(13), toInt32(-1)); +SELECT round(toUInt16(13), toInt64(-1)), ceil(toUInt16(13), toInt64(-1)), floor(toUInt16(13), toInt64(-1)); +SELECT round(toUInt16(13), toFloat32(1.1)), ceil(toUInt16(13), toFloat32(-1.1)), floor(toUInt16(13), toFloat32(-1.1)); +SELECT round(toUInt16(13), toFloat64(1.1)), ceil(toUInt16(13), toFloat64(-1.1)), floor(toUInt16(13), toFloat64(-1.1)); + +SELECT round(toUInt16(13), toInt8(-2)), ceil(toUInt16(13), toInt8(-2)), floor(toUInt16(13), toInt8(-2)); +SELECT round(toUInt16(13), toInt16(-2)), ceil(toUInt16(13), toInt16(-2)), floor(toUInt16(13), toInt16(-2)); +SELECT round(toUInt16(13), toInt32(-2)), ceil(toUInt16(13), toInt32(-2)), floor(toUInt16(13), toInt32(-2)); +SELECT round(toUInt16(13), toInt64(-2)), ceil(toUInt16(13), toInt64(-2)), floor(toUInt16(13), toInt64(-2)); +SELECT round(toUInt16(13), toFloat32(-2.1)), ceil(toUInt16(13), toFloat32(-2.1)), floor(toUInt16(13), toFloat32(-2.1)); +SELECT round(toUInt16(13), toFloat64(-2.1)), ceil(toUInt16(13), toFloat64(-2.1)), floor(toUInt16(13), toFloat64(-2.1)); + +/* UInt32 */ + +SELECT round(toUInt32(13), toUInt8(2)), ceil(toUInt32(13), toUInt8(2)), floor(toUInt32(13), toUInt8(2)); +SELECT round(toUInt32(13), toUInt16(2)), ceil(toUInt32(13), toUInt16(2)), floor(toUInt32(13), toUInt16(2)); +SELECT round(toUInt32(13), toUInt32(2)), ceil(toUInt32(13), toUInt32(2)), floor(toUInt32(13), toUInt32(2)); +SELECT round(toUInt32(13), toUInt64(2)), ceil(toUInt32(13), toUInt64(2)), floor(toUInt32(13), toUInt64(2)); +SELECT round(toUInt32(13), toInt8(2)), ceil(toUInt32(13), toInt8(2)), floor(toUInt32(13), toInt8(2)); +SELECT round(toUInt32(13), toInt16(2)), ceil(toUInt32(13), toInt16(2)), floor(toUInt32(13), toInt16(2)); +SELECT round(toUInt32(13), toInt32(2)), ceil(toUInt32(13), toInt32(2)), floor(toUInt32(13), toInt32(2)); +SELECT round(toUInt32(13), toInt64(2)), ceil(toUInt32(13), toInt64(2)), floor(toUInt32(13), toInt64(2)); +SELECT round(toUInt32(13), toFloat32(2.1)), ceil(toUInt32(13), toFloat32(2.1)), floor(toUInt32(13), toFloat32(2.1)); +SELECT round(toUInt32(13), toFloat64(2.1)), ceil(toUInt32(13), toFloat64(2.1)), floor(toUInt32(13), toFloat64(2.1)); + +SELECT round(toUInt32(13), toUInt8(1)), ceil(toUInt32(13), toUInt8(1)), floor(toUInt32(13), toUInt8(1)); +SELECT round(toUInt32(13), toUInt16(1)), ceil(toUInt32(13), toUInt16(1)), floor(toUInt32(13), toUInt16(1)); +SELECT round(toUInt32(13), toUInt32(1)), ceil(toUInt32(13), toUInt32(1)), floor(toUInt32(13), toUInt32(1)); +SELECT round(toUInt32(13), toUInt64(1)), ceil(toUInt32(13), toUInt64(1)), floor(toUInt32(13), toUInt64(1)); +SELECT round(toUInt32(13), toInt8(1)), ceil(toUInt32(13), toInt8(1)), floor(toUInt32(13), toInt8(1)); +SELECT round(toUInt32(13), toInt16(1)), ceil(toUInt32(13), toInt16(1)), floor(toUInt32(13), toInt16(1)); +SELECT round(toUInt32(13), toInt32(1)), ceil(toUInt32(13), toInt32(1)), floor(toUInt32(13), toInt32(1)); +SELECT round(toUInt32(13), toInt64(1)), ceil(toUInt32(13), toInt64(1)), floor(toUInt32(13), toInt64(1)); +SELECT round(toUInt32(13), toFloat32(1.1)), ceil(toUInt32(13), toFloat32(1.1)), floor(toUInt32(13), toFloat32(1.1)); +SELECT round(toUInt32(13), toFloat64(1.1)), ceil(toUInt32(13), toFloat64(1.1)), floor(toUInt32(13), toFloat64(1.1)); + +SELECT round(toUInt32(13), toUInt16(0)), ceil(toUInt32(13), toUInt16(0)), floor(toUInt32(13), toUInt16(0)); +SELECT round(toUInt32(13), toUInt32(0)), ceil(toUInt32(13), toUInt32(0)), floor(toUInt32(13), toUInt32(0)); +SELECT round(toUInt32(13), toUInt64(0)), ceil(toUInt32(13), toUInt64(0)), floor(toUInt32(13), toUInt64(0)); +SELECT round(toUInt32(13), toInt8(0)), ceil(toUInt32(13), toInt8(0)), floor(toUInt32(13), toInt8(0)); +SELECT round(toUInt32(13), toInt16(0)), ceil(toUInt32(13), toInt16(0)), floor(toUInt32(13), toInt16(0)); +SELECT round(toUInt32(13), toInt32(0)), ceil(toUInt32(13), toInt32(0)), floor(toUInt32(13), toInt32(0)); +SELECT round(toUInt32(13), toInt64(0)), ceil(toUInt32(13), toInt64(0)), floor(toUInt32(13), toInt64(0)); +SELECT round(toUInt32(13), toFloat32(0.1)), ceil(toUInt32(13), toFloat32(0.1)), floor(toUInt32(13), toFloat32(0.1)); +SELECT round(toUInt32(13), toFloat64(0.1)), ceil(toUInt32(13), toFloat64(0.1)), floor(toUInt32(13), toFloat64(0.1)); + +SELECT round(toUInt32(13), toInt8(-1)), ceil(toUInt32(13), toInt8(-1)), floor(toUInt32(13), toInt8(-1)); +SELECT round(toUInt32(13), toInt16(-1)), ceil(toUInt32(13), toInt16(-1)), floor(toUInt32(13), toInt16(-1)); +SELECT round(toUInt32(13), toInt32(-1)), ceil(toUInt32(13), toInt32(-1)), floor(toUInt32(13), toInt32(-1)); +SELECT round(toUInt32(13), toInt64(-1)), ceil(toUInt32(13), toInt64(-1)), floor(toUInt32(13), toInt64(-1)); +SELECT round(toUInt32(13), toFloat32(1.1)), ceil(toUInt32(13), toFloat32(-1.1)), floor(toUInt32(13), toFloat32(-1.1)); +SELECT round(toUInt32(13), toFloat64(1.1)), ceil(toUInt32(13), toFloat64(-1.1)), floor(toUInt32(13), toFloat64(-1.1)); + +SELECT round(toUInt32(13), toInt8(-2)), ceil(toUInt32(13), toInt8(-2)), floor(toUInt32(13), toInt8(-2)); +SELECT round(toUInt32(13), toInt16(-2)), ceil(toUInt32(13), toInt16(-2)), floor(toUInt32(13), toInt16(-2)); +SELECT round(toUInt32(13), toInt32(-2)), ceil(toUInt32(13), toInt32(-2)), floor(toUInt32(13), toInt32(-2)); +SELECT round(toUInt32(13), toInt64(-2)), ceil(toUInt32(13), toInt64(-2)), floor(toUInt32(13), toInt64(-2)); +SELECT round(toUInt32(13), toFloat32(-2.1)), ceil(toUInt32(13), toFloat32(-2.1)), floor(toUInt32(13), toFloat32(-2.1)); +SELECT round(toUInt32(13), toFloat64(-2.1)), ceil(toUInt32(13), toFloat64(-2.1)), floor(toUInt32(13), toFloat64(-2.1)); + +/* UInt64 */ + +SELECT round(toUInt64(13), toUInt8(2)), ceil(toUInt64(13), toUInt8(2)), floor(toUInt64(13), toUInt8(2)); +SELECT round(toUInt64(13), toUInt16(2)), ceil(toUInt64(13), toUInt16(2)), floor(toUInt64(13), toUInt16(2)); +SELECT round(toUInt64(13), toUInt32(2)), ceil(toUInt64(13), toUInt32(2)), floor(toUInt64(13), toUInt32(2)); +SELECT round(toUInt64(13), toUInt64(2)), ceil(toUInt64(13), toUInt64(2)), floor(toUInt64(13), toUInt64(2)); +SELECT round(toUInt64(13), toInt8(2)), ceil(toUInt64(13), toInt8(2)), floor(toUInt64(13), toInt8(2)); +SELECT round(toUInt64(13), toInt16(2)), ceil(toUInt64(13), toInt16(2)), floor(toUInt64(13), toInt16(2)); +SELECT round(toUInt64(13), toInt32(2)), ceil(toUInt64(13), toInt32(2)), floor(toUInt64(13), toInt32(2)); +SELECT round(toUInt64(13), toInt64(2)), ceil(toUInt64(13), toInt64(2)), floor(toUInt64(13), toInt64(2)); +SELECT round(toUInt64(13), toFloat32(2.1)), ceil(toUInt64(13), toFloat32(2.1)), floor(toUInt64(13), toFloat32(2.1)); +SELECT round(toUInt64(13), toFloat64(2.1)), ceil(toUInt64(13), toFloat64(2.1)), floor(toUInt64(13), toFloat64(2.1)); + +SELECT round(toUInt64(13), toUInt8(1)), ceil(toUInt64(13), toUInt8(1)), floor(toUInt64(13), toUInt8(1)); +SELECT round(toUInt64(13), toUInt16(1)), ceil(toUInt64(13), toUInt16(1)), floor(toUInt64(13), toUInt16(1)); +SELECT round(toUInt64(13), toUInt32(1)), ceil(toUInt64(13), toUInt32(1)), floor(toUInt64(13), toUInt32(1)); +SELECT round(toUInt64(13), toUInt64(1)), ceil(toUInt64(13), toUInt64(1)), floor(toUInt64(13), toUInt64(1)); +SELECT round(toUInt64(13), toInt8(1)), ceil(toUInt64(13), toInt8(1)), floor(toUInt64(13), toInt8(1)); +SELECT round(toUInt64(13), toInt16(1)), ceil(toUInt64(13), toInt16(1)), floor(toUInt64(13), toInt16(1)); +SELECT round(toUInt64(13), toInt32(1)), ceil(toUInt64(13), toInt32(1)), floor(toUInt64(13), toInt32(1)); +SELECT round(toUInt64(13), toInt64(1)), ceil(toUInt64(13), toInt64(1)), floor(toUInt64(13), toInt64(1)); +SELECT round(toUInt64(13), toFloat32(1.1)), ceil(toUInt64(13), toFloat32(1.1)), floor(toUInt64(13), toFloat32(1.1)); +SELECT round(toUInt64(13), toFloat64(1.1)), ceil(toUInt64(13), toFloat64(1.1)), floor(toUInt64(13), toFloat64(1.1)); + +SELECT round(toUInt64(13), toUInt16(0)), ceil(toUInt64(13), toUInt16(0)), floor(toUInt64(13), toUInt16(0)); +SELECT round(toUInt64(13), toUInt32(0)), ceil(toUInt64(13), toUInt32(0)), floor(toUInt64(13), toUInt32(0)); +SELECT round(toUInt64(13), toUInt64(0)), ceil(toUInt64(13), toUInt64(0)), floor(toUInt64(13), toUInt64(0)); +SELECT round(toUInt64(13), toInt8(0)), ceil(toUInt64(13), toInt8(0)), floor(toUInt64(13), toInt8(0)); +SELECT round(toUInt64(13), toInt16(0)), ceil(toUInt64(13), toInt16(0)), floor(toUInt64(13), toInt16(0)); +SELECT round(toUInt64(13), toInt32(0)), ceil(toUInt64(13), toInt32(0)), floor(toUInt64(13), toInt32(0)); +SELECT round(toUInt64(13), toInt64(0)), ceil(toUInt64(13), toInt64(0)), floor(toUInt64(13), toInt64(0)); +SELECT round(toUInt64(13), toFloat32(0.1)), ceil(toUInt64(13), toFloat32(0.1)), floor(toUInt64(13), toFloat32(0.1)); +SELECT round(toUInt64(13), toFloat64(0.1)), ceil(toUInt64(13), toFloat64(0.1)), floor(toUInt64(13), toFloat64(0.1)); + +SELECT round(toUInt64(13), toInt8(-1)), ceil(toUInt64(13), toInt8(-1)), floor(toUInt64(13), toInt8(-1)); +SELECT round(toUInt64(13), toInt16(-1)), ceil(toUInt64(13), toInt16(-1)), floor(toUInt64(13), toInt16(-1)); +SELECT round(toUInt64(13), toInt32(-1)), ceil(toUInt64(13), toInt32(-1)), floor(toUInt64(13), toInt32(-1)); +SELECT round(toUInt64(13), toInt64(-1)), ceil(toUInt64(13), toInt64(-1)), floor(toUInt64(13), toInt64(-1)); +SELECT round(toUInt64(13), toFloat32(1.1)), ceil(toUInt64(13), toFloat32(-1.1)), floor(toUInt64(13), toFloat32(-1.1)); +SELECT round(toUInt64(13), toFloat64(1.1)), ceil(toUInt64(13), toFloat64(-1.1)), floor(toUInt64(13), toFloat64(-1.1)); + +SELECT round(toUInt64(13), toInt8(-2)), ceil(toUInt64(13), toInt8(-2)), floor(toUInt64(13), toInt8(-2)); +SELECT round(toUInt64(13), toInt16(-2)), ceil(toUInt64(13), toInt16(-2)), floor(toUInt64(13), toInt16(-2)); +SELECT round(toUInt64(13), toInt32(-2)), ceil(toUInt64(13), toInt32(-2)), floor(toUInt64(13), toInt32(-2)); +SELECT round(toUInt64(13), toInt64(-2)), ceil(toUInt64(13), toInt64(-2)), floor(toUInt64(13), toInt64(-2)); +SELECT round(toUInt64(13), toFloat32(-2.1)), ceil(toUInt64(13), toFloat32(-2.1)), floor(toUInt64(13), toFloat32(-2.1)); +SELECT round(toUInt64(13), toFloat64(-2.1)), ceil(toUInt64(13), toFloat64(-2.1)), floor(toUInt64(13), toFloat64(-2.1)); + +/* Int8 */ + +SELECT round(toInt8(13), toUInt8(2)), ceil(toInt8(13), toUInt8(2)), floor(toInt8(13), toUInt8(2)); +SELECT round(toInt8(13), toUInt16(2)), ceil(toInt8(13), toUInt16(2)), floor(toInt8(13), toUInt16(2)); +SELECT round(toInt8(13), toUInt32(2)), ceil(toInt8(13), toUInt32(2)), floor(toInt8(13), toUInt32(2)); +SELECT round(toInt8(13), toUInt64(2)), ceil(toInt8(13), toUInt64(2)), floor(toInt8(13), toUInt64(2)); +SELECT round(toInt8(13), toInt8(2)), ceil(toInt8(13), toInt8(2)), floor(toInt8(13), toInt8(2)); +SELECT round(toInt8(13), toInt16(2)), ceil(toInt8(13), toInt16(2)), floor(toInt8(13), toInt16(2)); +SELECT round(toInt8(13), toInt32(2)), ceil(toInt8(13), toInt32(2)), floor(toInt8(13), toInt32(2)); +SELECT round(toInt8(13), toInt64(2)), ceil(toInt8(13), toInt64(2)), floor(toInt8(13), toInt64(2)); +SELECT round(toInt8(13), toFloat32(2.1)), ceil(toInt8(13), toFloat32(2.1)), floor(toInt8(13), toFloat32(2.1)); +SELECT round(toInt8(13), toFloat64(2.1)), ceil(toInt8(13), toFloat64(2.1)), floor(toInt8(13), toFloat64(2.1)); + +SELECT round(toInt8(13), toUInt8(1)), ceil(toInt8(13), toUInt8(1)), floor(toInt8(13), toUInt8(1)); +SELECT round(toInt8(13), toUInt16(1)), ceil(toInt8(13), toUInt16(1)), floor(toInt8(13), toUInt16(1)); +SELECT round(toInt8(13), toUInt32(1)), ceil(toInt8(13), toUInt32(1)), floor(toInt8(13), toUInt32(1)); +SELECT round(toInt8(13), toUInt64(1)), ceil(toInt8(13), toUInt64(1)), floor(toInt8(13), toUInt64(1)); +SELECT round(toInt8(13), toInt8(1)), ceil(toInt8(13), toInt8(1)), floor(toInt8(13), toInt8(1)); +SELECT round(toInt8(13), toInt16(1)), ceil(toInt8(13), toInt16(1)), floor(toInt8(13), toInt16(1)); +SELECT round(toInt8(13), toInt32(1)), ceil(toInt8(13), toInt32(1)), floor(toInt8(13), toInt32(1)); +SELECT round(toInt8(13), toInt64(1)), ceil(toInt8(13), toInt64(1)), floor(toInt8(13), toInt64(1)); +SELECT round(toInt8(13), toFloat32(1.1)), ceil(toInt8(13), toFloat32(1.1)), floor(toInt8(13), toFloat32(1.1)); +SELECT round(toInt8(13), toFloat64(1.1)), ceil(toInt8(13), toFloat64(1.1)), floor(toInt8(13), toFloat64(1.1)); + +SELECT round(toInt8(13), toUInt16(0)), ceil(toInt8(13), toUInt16(0)), floor(toInt8(13), toUInt16(0)); +SELECT round(toInt8(13), toUInt32(0)), ceil(toInt8(13), toUInt32(0)), floor(toInt8(13), toUInt32(0)); +SELECT round(toInt8(13), toUInt64(0)), ceil(toInt8(13), toUInt64(0)), floor(toInt8(13), toUInt64(0)); +SELECT round(toInt8(13), toInt8(0)), ceil(toInt8(13), toInt8(0)), floor(toInt8(13), toInt8(0)); +SELECT round(toInt8(13), toInt16(0)), ceil(toInt8(13), toInt16(0)), floor(toInt8(13), toInt16(0)); +SELECT round(toInt8(13), toInt32(0)), ceil(toInt8(13), toInt32(0)), floor(toInt8(13), toInt32(0)); +SELECT round(toInt8(13), toInt64(0)), ceil(toInt8(13), toInt64(0)), floor(toInt8(13), toInt64(0)); +SELECT round(toInt8(13), toFloat32(0.1)), ceil(toInt8(13), toFloat32(0.1)), floor(toInt8(13), toFloat32(0.1)); +SELECT round(toInt8(13), toFloat64(0.1)), ceil(toInt8(13), toFloat64(0.1)), floor(toInt8(13), toFloat64(0.1)); + +SELECT round(toInt8(13), toInt8(-1)), ceil(toInt8(13), toInt8(-1)), floor(toInt8(13), toInt8(-1)); +SELECT round(toInt8(13), toInt16(-1)), ceil(toInt8(13), toInt16(-1)), floor(toInt8(13), toInt16(-1)); +SELECT round(toInt8(13), toInt32(-1)), ceil(toInt8(13), toInt32(-1)), floor(toInt8(13), toInt32(-1)); +SELECT round(toInt8(13), toInt64(-1)), ceil(toInt8(13), toInt64(-1)), floor(toInt8(13), toInt64(-1)); +SELECT round(toInt8(13), toFloat32(1.1)), ceil(toInt8(13), toFloat32(-1.1)), floor(toInt8(13), toFloat32(-1.1)); +SELECT round(toInt8(13), toFloat64(1.1)), ceil(toInt8(13), toFloat64(-1.1)), floor(toInt8(13), toFloat64(-1.1)); + +SELECT round(toInt8(13), toInt8(-2)), ceil(toInt8(13), toInt8(-2)), floor(toInt8(13), toInt8(-2)); +SELECT round(toInt8(13), toInt16(-2)), ceil(toInt8(13), toInt16(-2)), floor(toInt8(13), toInt16(-2)); +SELECT round(toInt8(13), toInt32(-2)), ceil(toInt8(13), toInt32(-2)), floor(toInt8(13), toInt32(-2)); +SELECT round(toInt8(13), toInt64(-2)), ceil(toInt8(13), toInt64(-2)), floor(toInt8(13), toInt64(-2)); +SELECT round(toInt8(13), toFloat32(-2.1)), ceil(toInt8(13), toFloat32(-2.1)), floor(toInt8(13), toFloat32(-2.1)); +SELECT round(toInt8(13), toFloat64(-2.1)), ceil(toInt8(13), toFloat64(-2.1)), floor(toInt8(13), toFloat64(-2.1)); + +/* Int16 */ + +SELECT round(toInt16(13), toUInt8(2)), ceil(toInt16(13), toUInt8(2)), floor(toInt16(13), toUInt8(2)); +SELECT round(toInt16(13), toUInt16(2)), ceil(toInt16(13), toUInt16(2)), floor(toInt16(13), toUInt16(2)); +SELECT round(toInt16(13), toUInt32(2)), ceil(toInt16(13), toUInt32(2)), floor(toInt16(13), toUInt32(2)); +SELECT round(toInt16(13), toUInt64(2)), ceil(toInt16(13), toUInt64(2)), floor(toInt16(13), toUInt64(2)); +SELECT round(toInt16(13), toInt8(2)), ceil(toInt16(13), toInt8(2)), floor(toInt16(13), toInt8(2)); +SELECT round(toInt16(13), toInt16(2)), ceil(toInt16(13), toInt16(2)), floor(toInt16(13), toInt16(2)); +SELECT round(toInt16(13), toInt32(2)), ceil(toInt16(13), toInt32(2)), floor(toInt16(13), toInt32(2)); +SELECT round(toInt16(13), toInt64(2)), ceil(toInt16(13), toInt64(2)), floor(toInt16(13), toInt64(2)); +SELECT round(toInt16(13), toFloat32(2.1)), ceil(toInt16(13), toFloat32(2.1)), floor(toInt16(13), toFloat32(2.1)); +SELECT round(toInt16(13), toFloat64(2.1)), ceil(toInt16(13), toFloat64(2.1)), floor(toInt16(13), toFloat64(2.1)); + +SELECT round(toInt16(13), toUInt8(1)), ceil(toInt16(13), toUInt8(1)), floor(toInt16(13), toUInt8(1)); +SELECT round(toInt16(13), toUInt16(1)), ceil(toInt16(13), toUInt16(1)), floor(toInt16(13), toUInt16(1)); +SELECT round(toInt16(13), toUInt32(1)), ceil(toInt16(13), toUInt32(1)), floor(toInt16(13), toUInt32(1)); +SELECT round(toInt16(13), toUInt64(1)), ceil(toInt16(13), toUInt64(1)), floor(toInt16(13), toUInt64(1)); +SELECT round(toInt16(13), toInt8(1)), ceil(toInt16(13), toInt8(1)), floor(toInt16(13), toInt8(1)); +SELECT round(toInt16(13), toInt16(1)), ceil(toInt16(13), toInt16(1)), floor(toInt16(13), toInt16(1)); +SELECT round(toInt16(13), toInt32(1)), ceil(toInt16(13), toInt32(1)), floor(toInt16(13), toInt32(1)); +SELECT round(toInt16(13), toInt64(1)), ceil(toInt16(13), toInt64(1)), floor(toInt16(13), toInt64(1)); +SELECT round(toInt16(13), toFloat32(1.1)), ceil(toInt16(13), toFloat32(1.1)), floor(toInt16(13), toFloat32(1.1)); +SELECT round(toInt16(13), toFloat64(1.1)), ceil(toInt16(13), toFloat64(1.1)), floor(toInt16(13), toFloat64(1.1)); + +SELECT round(toInt16(13), toUInt16(0)), ceil(toInt16(13), toUInt16(0)), floor(toInt16(13), toUInt16(0)); +SELECT round(toInt16(13), toUInt32(0)), ceil(toInt16(13), toUInt32(0)), floor(toInt16(13), toUInt32(0)); +SELECT round(toInt16(13), toUInt64(0)), ceil(toInt16(13), toUInt64(0)), floor(toInt16(13), toUInt64(0)); +SELECT round(toInt16(13), toInt8(0)), ceil(toInt16(13), toInt8(0)), floor(toInt16(13), toInt8(0)); +SELECT round(toInt16(13), toInt16(0)), ceil(toInt16(13), toInt16(0)), floor(toInt16(13), toInt16(0)); +SELECT round(toInt16(13), toInt32(0)), ceil(toInt16(13), toInt32(0)), floor(toInt16(13), toInt32(0)); +SELECT round(toInt16(13), toInt64(0)), ceil(toInt16(13), toInt64(0)), floor(toInt16(13), toInt64(0)); +SELECT round(toInt16(13), toFloat32(0.1)), ceil(toInt16(13), toFloat32(0.1)), floor(toInt16(13), toFloat32(0.1)); +SELECT round(toInt16(13), toFloat64(0.1)), ceil(toInt16(13), toFloat64(0.1)), floor(toInt16(13), toFloat64(0.1)); + +SELECT round(toInt16(13), toInt8(-1)), ceil(toInt16(13), toInt8(-1)), floor(toInt16(13), toInt8(-1)); +SELECT round(toInt16(13), toInt16(-1)), ceil(toInt16(13), toInt16(-1)), floor(toInt16(13), toInt16(-1)); +SELECT round(toInt16(13), toInt32(-1)), ceil(toInt16(13), toInt32(-1)), floor(toInt16(13), toInt32(-1)); +SELECT round(toInt16(13), toInt64(-1)), ceil(toInt16(13), toInt64(-1)), floor(toInt16(13), toInt64(-1)); +SELECT round(toInt16(13), toFloat32(1.1)), ceil(toInt16(13), toFloat32(-1.1)), floor(toInt16(13), toFloat32(-1.1)); +SELECT round(toInt16(13), toFloat64(1.1)), ceil(toInt16(13), toFloat64(-1.1)), floor(toInt16(13), toFloat64(-1.1)); + +SELECT round(toInt16(13), toInt8(-2)), ceil(toInt16(13), toInt8(-2)), floor(toInt16(13), toInt8(-2)); +SELECT round(toInt16(13), toInt16(-2)), ceil(toInt16(13), toInt16(-2)), floor(toInt16(13), toInt16(-2)); +SELECT round(toInt16(13), toInt32(-2)), ceil(toInt16(13), toInt32(-2)), floor(toInt16(13), toInt32(-2)); +SELECT round(toInt16(13), toInt64(-2)), ceil(toInt16(13), toInt64(-2)), floor(toInt16(13), toInt64(-2)); +SELECT round(toInt16(13), toFloat32(-2.1)), ceil(toInt16(13), toFloat32(-2.1)), floor(toInt16(13), toFloat32(-2.1)); +SELECT round(toInt16(13), toFloat64(-2.1)), ceil(toInt16(13), toFloat64(-2.1)), floor(toInt16(13), toFloat64(-2.1)); + +/* Int32 */ + +SELECT round(toInt32(13), toUInt8(2)), ceil(toInt32(13), toUInt8(2)), floor(toInt32(13), toUInt8(2)); +SELECT round(toInt32(13), toUInt16(2)), ceil(toInt32(13), toUInt16(2)), floor(toInt32(13), toUInt16(2)); +SELECT round(toInt32(13), toUInt32(2)), ceil(toInt32(13), toUInt32(2)), floor(toInt32(13), toUInt32(2)); +SELECT round(toInt32(13), toUInt64(2)), ceil(toInt32(13), toUInt64(2)), floor(toInt32(13), toUInt64(2)); +SELECT round(toInt32(13), toInt8(2)), ceil(toInt32(13), toInt8(2)), floor(toInt32(13), toInt8(2)); +SELECT round(toInt32(13), toInt16(2)), ceil(toInt32(13), toInt16(2)), floor(toInt32(13), toInt16(2)); +SELECT round(toInt32(13), toInt32(2)), ceil(toInt32(13), toInt32(2)), floor(toInt32(13), toInt32(2)); +SELECT round(toInt32(13), toInt64(2)), ceil(toInt32(13), toInt64(2)), floor(toInt32(13), toInt64(2)); +SELECT round(toInt32(13), toFloat32(2.1)), ceil(toInt32(13), toFloat32(2.1)), floor(toInt32(13), toFloat32(2.1)); +SELECT round(toInt32(13), toFloat64(2.1)), ceil(toInt32(13), toFloat64(2.1)), floor(toInt32(13), toFloat64(2.1)); + +SELECT round(toInt32(13), toUInt8(1)), ceil(toInt32(13), toUInt8(1)), floor(toInt32(13), toUInt8(1)); +SELECT round(toInt32(13), toUInt16(1)), ceil(toInt32(13), toUInt16(1)), floor(toInt32(13), toUInt16(1)); +SELECT round(toInt32(13), toUInt32(1)), ceil(toInt32(13), toUInt32(1)), floor(toInt32(13), toUInt32(1)); +SELECT round(toInt32(13), toUInt64(1)), ceil(toInt32(13), toUInt64(1)), floor(toInt32(13), toUInt64(1)); +SELECT round(toInt32(13), toInt8(1)), ceil(toInt32(13), toInt8(1)), floor(toInt32(13), toInt8(1)); +SELECT round(toInt32(13), toInt16(1)), ceil(toInt32(13), toInt16(1)), floor(toInt32(13), toInt16(1)); +SELECT round(toInt32(13), toInt32(1)), ceil(toInt32(13), toInt32(1)), floor(toInt32(13), toInt32(1)); +SELECT round(toInt32(13), toInt64(1)), ceil(toInt32(13), toInt64(1)), floor(toInt32(13), toInt64(1)); +SELECT round(toInt32(13), toFloat32(1.1)), ceil(toInt32(13), toFloat32(1.1)), floor(toInt32(13), toFloat32(1.1)); +SELECT round(toInt32(13), toFloat64(1.1)), ceil(toInt32(13), toFloat64(1.1)), floor(toInt32(13), toFloat64(1.1)); + +SELECT round(toInt32(13), toUInt16(0)), ceil(toInt32(13), toUInt16(0)), floor(toInt32(13), toUInt16(0)); +SELECT round(toInt32(13), toUInt32(0)), ceil(toInt32(13), toUInt32(0)), floor(toInt32(13), toUInt32(0)); +SELECT round(toInt32(13), toUInt64(0)), ceil(toInt32(13), toUInt64(0)), floor(toInt32(13), toUInt64(0)); +SELECT round(toInt32(13), toInt8(0)), ceil(toInt32(13), toInt8(0)), floor(toInt32(13), toInt8(0)); +SELECT round(toInt32(13), toInt16(0)), ceil(toInt32(13), toInt16(0)), floor(toInt32(13), toInt16(0)); +SELECT round(toInt32(13), toInt32(0)), ceil(toInt32(13), toInt32(0)), floor(toInt32(13), toInt32(0)); +SELECT round(toInt32(13), toInt64(0)), ceil(toInt32(13), toInt64(0)), floor(toInt32(13), toInt64(0)); +SELECT round(toInt32(13), toFloat32(0.1)), ceil(toInt32(13), toFloat32(0.1)), floor(toInt32(13), toFloat32(0.1)); +SELECT round(toInt32(13), toFloat64(0.1)), ceil(toInt32(13), toFloat64(0.1)), floor(toInt32(13), toFloat64(0.1)); + +SELECT round(toInt32(13), toInt8(-1)), ceil(toInt32(13), toInt8(-1)), floor(toInt32(13), toInt8(-1)); +SELECT round(toInt32(13), toInt16(-1)), ceil(toInt32(13), toInt16(-1)), floor(toInt32(13), toInt16(-1)); +SELECT round(toInt32(13), toInt32(-1)), ceil(toInt32(13), toInt32(-1)), floor(toInt32(13), toInt32(-1)); +SELECT round(toInt32(13), toInt64(-1)), ceil(toInt32(13), toInt64(-1)), floor(toInt32(13), toInt64(-1)); +SELECT round(toInt32(13), toFloat32(1.1)), ceil(toInt32(13), toFloat32(-1.1)), floor(toInt32(13), toFloat32(-1.1)); +SELECT round(toInt32(13), toFloat64(1.1)), ceil(toInt32(13), toFloat64(-1.1)), floor(toInt32(13), toFloat64(-1.1)); + +SELECT round(toInt32(13), toInt8(-2)), ceil(toInt32(13), toInt8(-2)), floor(toInt32(13), toInt8(-2)); +SELECT round(toInt32(13), toInt16(-2)), ceil(toInt32(13), toInt16(-2)), floor(toInt32(13), toInt16(-2)); +SELECT round(toInt32(13), toInt32(-2)), ceil(toInt32(13), toInt32(-2)), floor(toInt32(13), toInt32(-2)); +SELECT round(toInt32(13), toInt64(-2)), ceil(toInt32(13), toInt64(-2)), floor(toInt32(13), toInt64(-2)); +SELECT round(toInt32(13), toFloat32(-2.1)), ceil(toInt32(13), toFloat32(-2.1)), floor(toInt32(13), toFloat32(-2.1)); +SELECT round(toInt32(13), toFloat64(-2.1)), ceil(toInt32(13), toFloat64(-2.1)), floor(toInt32(13), toFloat64(-2.1)); + +/* Int64 */ + +SELECT round(toInt64(13), toUInt8(2)), ceil(toInt64(13), toUInt8(2)), floor(toInt64(13), toUInt8(2)); +SELECT round(toInt64(13), toUInt16(2)), ceil(toInt64(13), toUInt16(2)), floor(toInt64(13), toUInt16(2)); +SELECT round(toInt64(13), toUInt32(2)), ceil(toInt64(13), toUInt32(2)), floor(toInt64(13), toUInt32(2)); +SELECT round(toInt64(13), toUInt64(2)), ceil(toInt64(13), toUInt64(2)), floor(toInt64(13), toUInt64(2)); +SELECT round(toInt64(13), toInt8(2)), ceil(toInt64(13), toInt8(2)), floor(toInt64(13), toInt8(2)); +SELECT round(toInt64(13), toInt16(2)), ceil(toInt64(13), toInt16(2)), floor(toInt64(13), toInt16(2)); +SELECT round(toInt64(13), toInt32(2)), ceil(toInt64(13), toInt32(2)), floor(toInt64(13), toInt32(2)); +SELECT round(toInt64(13), toInt64(2)), ceil(toInt64(13), toInt64(2)), floor(toInt64(13), toInt64(2)); +SELECT round(toInt64(13), toFloat32(2.1)), ceil(toInt64(13), toFloat32(2.1)), floor(toInt64(13), toFloat32(2.1)); +SELECT round(toInt64(13), toFloat64(2.1)), ceil(toInt64(13), toFloat64(2.1)), floor(toInt64(13), toFloat64(2.1)); + +SELECT round(toInt64(13), toUInt8(1)), ceil(toInt64(13), toUInt8(1)), floor(toInt64(13), toUInt8(1)); +SELECT round(toInt64(13), toUInt16(1)), ceil(toInt64(13), toUInt16(1)), floor(toInt64(13), toUInt16(1)); +SELECT round(toInt64(13), toUInt32(1)), ceil(toInt64(13), toUInt32(1)), floor(toInt64(13), toUInt32(1)); +SELECT round(toInt64(13), toUInt64(1)), ceil(toInt64(13), toUInt64(1)), floor(toInt64(13), toUInt64(1)); +SELECT round(toInt64(13), toInt8(1)), ceil(toInt64(13), toInt8(1)), floor(toInt64(13), toInt8(1)); +SELECT round(toInt64(13), toInt16(1)), ceil(toInt64(13), toInt16(1)), floor(toInt64(13), toInt16(1)); +SELECT round(toInt64(13), toInt32(1)), ceil(toInt64(13), toInt32(1)), floor(toInt64(13), toInt32(1)); +SELECT round(toInt64(13), toInt64(1)), ceil(toInt64(13), toInt64(1)), floor(toInt64(13), toInt64(1)); +SELECT round(toInt64(13), toFloat32(1.1)), ceil(toInt64(13), toFloat32(1.1)), floor(toInt64(13), toFloat32(1.1)); +SELECT round(toInt64(13), toFloat64(1.1)), ceil(toInt64(13), toFloat64(1.1)), floor(toInt64(13), toFloat64(1.1)); + +SELECT round(toInt64(13), toUInt16(0)), ceil(toInt64(13), toUInt16(0)), floor(toInt64(13), toUInt16(0)); +SELECT round(toInt64(13), toUInt32(0)), ceil(toInt64(13), toUInt32(0)), floor(toInt64(13), toUInt32(0)); +SELECT round(toInt64(13), toUInt64(0)), ceil(toInt64(13), toUInt64(0)), floor(toInt64(13), toUInt64(0)); +SELECT round(toInt64(13), toInt8(0)), ceil(toInt64(13), toInt8(0)), floor(toInt64(13), toInt8(0)); +SELECT round(toInt64(13), toInt16(0)), ceil(toInt64(13), toInt16(0)), floor(toInt64(13), toInt16(0)); +SELECT round(toInt64(13), toInt32(0)), ceil(toInt64(13), toInt32(0)), floor(toInt64(13), toInt32(0)); +SELECT round(toInt64(13), toInt64(0)), ceil(toInt64(13), toInt64(0)), floor(toInt64(13), toInt64(0)); +SELECT round(toInt64(13), toFloat32(0.1)), ceil(toInt64(13), toFloat32(0.1)), floor(toInt64(13), toFloat32(0.1)); +SELECT round(toInt64(13), toFloat64(0.1)), ceil(toInt64(13), toFloat64(0.1)), floor(toInt64(13), toFloat64(0.1)); + +SELECT round(toInt64(13), toInt8(-1)), ceil(toInt64(13), toInt8(-1)), floor(toInt64(13), toInt8(-1)); +SELECT round(toInt64(13), toInt16(-1)), ceil(toInt64(13), toInt16(-1)), floor(toInt64(13), toInt16(-1)); +SELECT round(toInt64(13), toInt32(-1)), ceil(toInt64(13), toInt32(-1)), floor(toInt64(13), toInt32(-1)); +SELECT round(toInt64(13), toInt64(-1)), ceil(toInt64(13), toInt64(-1)), floor(toInt64(13), toInt64(-1)); +SELECT round(toInt64(13), toFloat32(1.1)), ceil(toInt64(13), toFloat32(-1.1)), floor(toInt64(13), toFloat32(-1.1)); +SELECT round(toInt64(13), toFloat64(1.1)), ceil(toInt64(13), toFloat64(-1.1)), floor(toInt64(13), toFloat64(-1.1)); + +SELECT round(toInt64(13), toInt8(-2)), ceil(toInt64(13), toInt8(-2)), floor(toInt64(13), toInt8(-2)); +SELECT round(toInt64(13), toInt16(-2)), ceil(toInt64(13), toInt16(-2)), floor(toInt64(13), toInt16(-2)); +SELECT round(toInt64(13), toInt32(-2)), ceil(toInt64(13), toInt32(-2)), floor(toInt64(13), toInt32(-2)); +SELECT round(toInt64(13), toInt64(-2)), ceil(toInt64(13), toInt64(-2)), floor(toInt64(13), toInt64(-2)); +SELECT round(toInt64(13), toFloat32(-2.1)), ceil(toInt64(13), toFloat32(-2.1)), floor(toInt64(13), toFloat32(-2.1)); +SELECT round(toInt64(13), toFloat64(-2.1)), ceil(toInt64(13), toFloat64(-2.1)), floor(toInt64(13), toFloat64(-2.1)); + +/* Float32 */ + +SELECT round(toFloat32(13), toUInt8(2)), ceil(toFloat32(13), toUInt8(2)), floor(toFloat32(13), toUInt8(2)); +SELECT round(toFloat32(13), toUInt16(2)), ceil(toFloat32(13), toUInt16(2)), floor(toFloat32(13), toUInt16(2)); +SELECT round(toFloat32(13), toUInt32(2)), ceil(toFloat32(13), toUInt32(2)), floor(toFloat32(13), toUInt32(2)); +SELECT round(toFloat32(13), toUInt64(2)), ceil(toFloat32(13), toUInt64(2)), floor(toFloat32(13), toUInt64(2)); +SELECT round(toFloat32(13), toInt8(2)), ceil(toFloat32(13), toInt8(2)), floor(toFloat32(13), toInt8(2)); +SELECT round(toFloat32(13), toInt16(2)), ceil(toFloat32(13), toInt16(2)), floor(toFloat32(13), toInt16(2)); +SELECT round(toFloat32(13), toInt32(2)), ceil(toFloat32(13), toInt32(2)), floor(toFloat32(13), toInt32(2)); +SELECT round(toFloat32(13), toInt64(2)), ceil(toFloat32(13), toInt64(2)), floor(toFloat32(13), toInt64(2)); +SELECT round(toFloat32(13), toFloat32(2.1)), ceil(toFloat32(13), toFloat32(2.1)), floor(toFloat32(13), toFloat32(2.1)); +SELECT round(toFloat32(13), toFloat64(2.1)), ceil(toFloat32(13), toFloat64(2.1)), floor(toFloat32(13), toFloat64(2.1)); + +SELECT round(toFloat32(13), toUInt8(1)), ceil(toFloat32(13), toUInt8(1)), floor(toFloat32(13), toUInt8(1)); +SELECT round(toFloat32(13), toUInt16(1)), ceil(toFloat32(13), toUInt16(1)), floor(toFloat32(13), toUInt16(1)); +SELECT round(toFloat32(13), toUInt32(1)), ceil(toFloat32(13), toUInt32(1)), floor(toFloat32(13), toUInt32(1)); +SELECT round(toFloat32(13), toUInt64(1)), ceil(toFloat32(13), toUInt64(1)), floor(toFloat32(13), toUInt64(1)); +SELECT round(toFloat32(13), toInt8(1)), ceil(toFloat32(13), toInt8(1)), floor(toFloat32(13), toInt8(1)); +SELECT round(toFloat32(13), toInt16(1)), ceil(toFloat32(13), toInt16(1)), floor(toFloat32(13), toInt16(1)); +SELECT round(toFloat32(13), toInt32(1)), ceil(toFloat32(13), toInt32(1)), floor(toFloat32(13), toInt32(1)); +SELECT round(toFloat32(13), toInt64(1)), ceil(toFloat32(13), toInt64(1)), floor(toFloat32(13), toInt64(1)); +SELECT round(toFloat32(13), toFloat32(1.1)), ceil(toFloat32(13), toFloat32(1.1)), floor(toFloat32(13), toFloat32(1.1)); +SELECT round(toFloat32(13), toFloat64(1.1)), ceil(toFloat32(13), toFloat64(1.1)), floor(toFloat32(13), toFloat64(1.1)); + +SELECT round(toFloat32(13), toUInt16(0)), ceil(toFloat32(13), toUInt16(0)), floor(toFloat32(13), toUInt16(0)); +SELECT round(toFloat32(13), toUInt32(0)), ceil(toFloat32(13), toUInt32(0)), floor(toFloat32(13), toUInt32(0)); +SELECT round(toFloat32(13), toUInt64(0)), ceil(toFloat32(13), toUInt64(0)), floor(toFloat32(13), toUInt64(0)); +SELECT round(toFloat32(13), toInt8(0)), ceil(toFloat32(13), toInt8(0)), floor(toFloat32(13), toInt8(0)); +SELECT round(toFloat32(13), toInt16(0)), ceil(toFloat32(13), toInt16(0)), floor(toFloat32(13), toInt16(0)); +SELECT round(toFloat32(13), toInt32(0)), ceil(toFloat32(13), toInt32(0)), floor(toFloat32(13), toInt32(0)); +SELECT round(toFloat32(13), toInt64(0)), ceil(toFloat32(13), toInt64(0)), floor(toFloat32(13), toInt64(0)); +SELECT round(toFloat32(13), toFloat32(0.1)), ceil(toFloat32(13), toFloat32(0.1)), floor(toFloat32(13), toFloat32(0.1)); +SELECT round(toFloat32(13), toFloat64(0.1)), ceil(toFloat32(13), toFloat64(0.1)), floor(toFloat32(13), toFloat64(0.1)); + +SELECT round(toFloat32(13), toInt8(-1)), ceil(toFloat32(13), toInt8(-1)), floor(toFloat32(13), toInt8(-1)); +SELECT round(toFloat32(13), toInt16(-1)), ceil(toFloat32(13), toInt16(-1)), floor(toFloat32(13), toInt16(-1)); +SELECT round(toFloat32(13), toInt32(-1)), ceil(toFloat32(13), toInt32(-1)), floor(toFloat32(13), toInt32(-1)); +SELECT round(toFloat32(13), toInt64(-1)), ceil(toFloat32(13), toInt64(-1)), floor(toFloat32(13), toInt64(-1)); +SELECT round(toFloat32(13), toFloat32(1.1)), ceil(toFloat32(13), toFloat32(-1.1)), floor(toFloat32(13), toFloat32(-1.1)); +SELECT round(toFloat32(13), toFloat64(1.1)), ceil(toFloat32(13), toFloat64(-1.1)), floor(toFloat32(13), toFloat64(-1.1)); + +SELECT round(toFloat32(13), toInt8(-2)), ceil(toFloat32(13), toInt8(-2)), floor(toFloat32(13), toInt8(-2)); +SELECT round(toFloat32(13), toInt16(-2)), ceil(toFloat32(13), toInt16(-2)), floor(toFloat32(13), toInt16(-2)); +SELECT round(toFloat32(13), toInt32(-2)), ceil(toFloat32(13), toInt32(-2)), floor(toFloat32(13), toInt32(-2)); +SELECT round(toFloat32(13), toInt64(-2)), ceil(toFloat32(13), toInt64(-2)), floor(toFloat32(13), toInt64(-2)); +SELECT round(toFloat32(13), toFloat32(-2.1)), ceil(toFloat32(13), toFloat32(-2.1)), floor(toFloat32(13), toFloat32(-2.1)); +SELECT round(toFloat32(13), toFloat64(-2.1)), ceil(toFloat32(13), toFloat64(-2.1)), floor(toFloat32(13), toFloat64(-2.1)); + +/* Float64 */ + +SELECT round(toFloat64(13), toUInt8(2)), ceil(toFloat64(13), toUInt8(2)), floor(toFloat64(13), toUInt8(2)); +SELECT round(toFloat64(13), toUInt16(2)), ceil(toFloat64(13), toUInt16(2)), floor(toFloat64(13), toUInt16(2)); +SELECT round(toFloat64(13), toUInt32(2)), ceil(toFloat64(13), toUInt32(2)), floor(toFloat64(13), toUInt32(2)); +SELECT round(toFloat64(13), toUInt64(2)), ceil(toFloat64(13), toUInt64(2)), floor(toFloat64(13), toUInt64(2)); +SELECT round(toFloat64(13), toInt8(2)), ceil(toFloat64(13), toInt8(2)), floor(toFloat64(13), toInt8(2)); +SELECT round(toFloat64(13), toInt16(2)), ceil(toFloat64(13), toInt16(2)), floor(toFloat64(13), toInt16(2)); +SELECT round(toFloat64(13), toInt32(2)), ceil(toFloat64(13), toInt32(2)), floor(toFloat64(13), toInt32(2)); +SELECT round(toFloat64(13), toInt64(2)), ceil(toFloat64(13), toInt64(2)), floor(toFloat64(13), toInt64(2)); +SELECT round(toFloat64(13), toFloat32(2.1)), ceil(toFloat64(13), toFloat32(2.1)), floor(toFloat64(13), toFloat32(2.1)); +SELECT round(toFloat64(13), toFloat64(2.1)), ceil(toFloat64(13), toFloat64(2.1)), floor(toFloat64(13), toFloat64(2.1)); + +SELECT round(toFloat64(13), toUInt8(1)), ceil(toFloat64(13), toUInt8(1)), floor(toFloat64(13), toUInt8(1)); +SELECT round(toFloat64(13), toUInt16(1)), ceil(toFloat64(13), toUInt16(1)), floor(toFloat64(13), toUInt16(1)); +SELECT round(toFloat64(13), toUInt32(1)), ceil(toFloat64(13), toUInt32(1)), floor(toFloat64(13), toUInt32(1)); +SELECT round(toFloat64(13), toUInt64(1)), ceil(toFloat64(13), toUInt64(1)), floor(toFloat64(13), toUInt64(1)); +SELECT round(toFloat64(13), toInt8(1)), ceil(toFloat64(13), toInt8(1)), floor(toFloat64(13), toInt8(1)); +SELECT round(toFloat64(13), toInt16(1)), ceil(toFloat64(13), toInt16(1)), floor(toFloat64(13), toInt16(1)); +SELECT round(toFloat64(13), toInt32(1)), ceil(toFloat64(13), toInt32(1)), floor(toFloat64(13), toInt32(1)); +SELECT round(toFloat64(13), toInt64(1)), ceil(toFloat64(13), toInt64(1)), floor(toFloat64(13), toInt64(1)); +SELECT round(toFloat64(13), toFloat32(1.1)), ceil(toFloat64(13), toFloat32(1.1)), floor(toFloat64(13), toFloat32(1.1)); +SELECT round(toFloat64(13), toFloat64(1.1)), ceil(toFloat64(13), toFloat64(1.1)), floor(toFloat64(13), toFloat64(1.1)); + +SELECT round(toFloat64(13), toUInt16(0)), ceil(toFloat64(13), toUInt16(0)), floor(toFloat64(13), toUInt16(0)); +SELECT round(toFloat64(13), toUInt32(0)), ceil(toFloat64(13), toUInt32(0)), floor(toFloat64(13), toUInt32(0)); +SELECT round(toFloat64(13), toUInt64(0)), ceil(toFloat64(13), toUInt64(0)), floor(toFloat64(13), toUInt64(0)); +SELECT round(toFloat64(13), toInt8(0)), ceil(toFloat64(13), toInt8(0)), floor(toFloat64(13), toInt8(0)); +SELECT round(toFloat64(13), toInt16(0)), ceil(toFloat64(13), toInt16(0)), floor(toFloat64(13), toInt16(0)); +SELECT round(toFloat64(13), toInt32(0)), ceil(toFloat64(13), toInt32(0)), floor(toFloat64(13), toInt32(0)); +SELECT round(toFloat64(13), toInt64(0)), ceil(toFloat64(13), toInt64(0)), floor(toFloat64(13), toInt64(0)); +SELECT round(toFloat64(13), toFloat32(0.1)), ceil(toFloat64(13), toFloat32(0.1)), floor(toFloat64(13), toFloat32(0.1)); +SELECT round(toFloat64(13), toFloat64(0.1)), ceil(toFloat64(13), toFloat64(0.1)), floor(toFloat64(13), toFloat64(0.1)); + +SELECT round(toFloat64(13), toInt8(-1)), ceil(toFloat64(13), toInt8(-1)), floor(toFloat64(13), toInt8(-1)); +SELECT round(toFloat64(13), toInt16(-1)), ceil(toFloat64(13), toInt16(-1)), floor(toFloat64(13), toInt16(-1)); +SELECT round(toFloat64(13), toInt32(-1)), ceil(toFloat64(13), toInt32(-1)), floor(toFloat64(13), toInt32(-1)); +SELECT round(toFloat64(13), toInt64(-1)), ceil(toFloat64(13), toInt64(-1)), floor(toFloat64(13), toInt64(-1)); +SELECT round(toFloat64(13), toFloat32(1.1)), ceil(toFloat64(13), toFloat32(-1.1)), floor(toFloat64(13), toFloat32(-1.1)); +SELECT round(toFloat64(13), toFloat64(1.1)), ceil(toFloat64(13), toFloat64(-1.1)), floor(toFloat64(13), toFloat64(-1.1)); + +SELECT round(toFloat64(13), toInt8(-2)), ceil(toFloat64(13), toInt8(-2)), floor(toFloat64(13), toInt8(-2)); +SELECT round(toFloat64(13), toInt16(-2)), ceil(toFloat64(13), toInt16(-2)), floor(toFloat64(13), toInt16(-2)); +SELECT round(toFloat64(13), toInt32(-2)), ceil(toFloat64(13), toInt32(-2)), floor(toFloat64(13), toInt32(-2)); +SELECT round(toFloat64(13), toInt64(-2)), ceil(toFloat64(13), toInt64(-2)), floor(toFloat64(13), toInt64(-2)); +SELECT round(toFloat64(13), toFloat32(-2.1)), ceil(toFloat64(13), toFloat32(-2.1)), floor(toFloat64(13), toFloat32(-2.1)); +SELECT round(toFloat64(13), toFloat64(-2.1)), ceil(toFloat64(13), toFloat64(-2.1)), floor(toFloat64(13), toFloat64(-2.1)); + +/* Отрицательное значение */ + +/* Int8 */ + +SELECT round(toInt8(-13), toUInt8(2)), ceil(toInt8(-13), toUInt8(2)), floor(toInt8(-13), toUInt8(2)); +SELECT round(toInt8(-13), toUInt16(2)), ceil(toInt8(-13), toUInt16(2)), floor(toInt8(-13), toUInt16(2)); +SELECT round(toInt8(-13), toUInt32(2)), ceil(toInt8(-13), toUInt32(2)), floor(toInt8(-13), toUInt32(2)); +SELECT round(toInt8(-13), toUInt64(2)), ceil(toInt8(-13), toUInt64(2)), floor(toInt8(-13), toUInt64(2)); +SELECT round(toInt8(-13), toInt8(2)), ceil(toInt8(-13), toInt8(2)), floor(toInt8(-13), toInt8(2)); +SELECT round(toInt8(-13), toInt16(2)), ceil(toInt8(-13), toInt16(2)), floor(toInt8(-13), toInt16(2)); +SELECT round(toInt8(-13), toInt32(2)), ceil(toInt8(-13), toInt32(2)), floor(toInt8(-13), toInt32(2)); +SELECT round(toInt8(-13), toInt64(2)), ceil(toInt8(-13), toInt64(2)), floor(toInt8(-13), toInt64(2)); +SELECT round(toInt8(-13), toFloat32(2.1)), ceil(toInt8(-13), toFloat32(2.1)), floor(toInt8(-13), toFloat32(2.1)); +SELECT round(toInt8(-13), toFloat64(2.1)), ceil(toInt8(-13), toFloat64(2.1)), floor(toInt8(-13), toFloat64(2.1)); + +SELECT round(toInt8(-13), toUInt8(1)), ceil(toInt8(-13), toUInt8(1)), floor(toInt8(-13), toUInt8(1)); +SELECT round(toInt8(-13), toUInt16(1)), ceil(toInt8(-13), toUInt16(1)), floor(toInt8(-13), toUInt16(1)); +SELECT round(toInt8(-13), toUInt32(1)), ceil(toInt8(-13), toUInt32(1)), floor(toInt8(-13), toUInt32(1)); +SELECT round(toInt8(-13), toUInt64(1)), ceil(toInt8(-13), toUInt64(1)), floor(toInt8(-13), toUInt64(1)); +SELECT round(toInt8(-13), toInt8(1)), ceil(toInt8(-13), toInt8(1)), floor(toInt8(-13), toInt8(1)); +SELECT round(toInt8(-13), toInt16(1)), ceil(toInt8(-13), toInt16(1)), floor(toInt8(-13), toInt16(1)); +SELECT round(toInt8(-13), toInt32(1)), ceil(toInt8(-13), toInt32(1)), floor(toInt8(-13), toInt32(1)); +SELECT round(toInt8(-13), toInt64(1)), ceil(toInt8(-13), toInt64(1)), floor(toInt8(-13), toInt64(1)); +SELECT round(toInt8(-13), toFloat32(1.1)), ceil(toInt8(-13), toFloat32(1.1)), floor(toInt8(-13), toFloat32(1.1)); +SELECT round(toInt8(-13), toFloat64(1.1)), ceil(toInt8(-13), toFloat64(1.1)), floor(toInt8(-13), toFloat64(1.1)); + +SELECT round(toInt8(-13), toUInt16(0)), ceil(toInt8(-13), toUInt16(0)), floor(toInt8(-13), toUInt16(0)); +SELECT round(toInt8(-13), toUInt32(0)), ceil(toInt8(-13), toUInt32(0)), floor(toInt8(-13), toUInt32(0)); +SELECT round(toInt8(-13), toUInt64(0)), ceil(toInt8(-13), toUInt64(0)), floor(toInt8(-13), toUInt64(0)); +SELECT round(toInt8(-13), toInt8(0)), ceil(toInt8(-13), toInt8(0)), floor(toInt8(-13), toInt8(0)); +SELECT round(toInt8(-13), toInt16(0)), ceil(toInt8(-13), toInt16(0)), floor(toInt8(-13), toInt16(0)); +SELECT round(toInt8(-13), toInt32(0)), ceil(toInt8(-13), toInt32(0)), floor(toInt8(-13), toInt32(0)); +SELECT round(toInt8(-13), toInt64(0)), ceil(toInt8(-13), toInt64(0)), floor(toInt8(-13), toInt64(0)); +SELECT round(toInt8(-13), toFloat32(0.1)), ceil(toInt8(-13), toFloat32(0.1)), floor(toInt8(-13), toFloat32(0.1)); +SELECT round(toInt8(-13), toFloat64(0.1)), ceil(toInt8(-13), toFloat64(0.1)), floor(toInt8(-13), toFloat64(0.1)); + +SELECT round(toInt8(-13), toInt8(-1)), ceil(toInt8(-13), toInt8(-1)), floor(toInt8(-13), toInt8(-1)); +SELECT round(toInt8(-13), toInt16(-1)), ceil(toInt8(-13), toInt16(-1)), floor(toInt8(-13), toInt16(-1)); +SELECT round(toInt8(-13), toInt32(-1)), ceil(toInt8(-13), toInt32(-1)), floor(toInt8(-13), toInt32(-1)); +SELECT round(toInt8(-13), toInt64(-1)), ceil(toInt8(-13), toInt64(-1)), floor(toInt8(-13), toInt64(-1)); +SELECT round(toInt8(-13), toFloat32(1.1)), ceil(toInt8(-13), toFloat32(-1.1)), floor(toInt8(-13), toFloat32(-1.1)); +SELECT round(toInt8(-13), toFloat64(1.1)), ceil(toInt8(-13), toFloat64(-1.1)), floor(toInt8(-13), toFloat64(-1.1)); + +SELECT round(toInt8(-13), toInt8(-2)), ceil(toInt8(-13), toInt8(-2)), floor(toInt8(-13), toInt8(-2)); +SELECT round(toInt8(-13), toInt16(-2)), ceil(toInt8(-13), toInt16(-2)), floor(toInt8(-13), toInt16(-2)); +SELECT round(toInt8(-13), toInt32(-2)), ceil(toInt8(-13), toInt32(-2)), floor(toInt8(-13), toInt32(-2)); +SELECT round(toInt8(-13), toInt64(-2)), ceil(toInt8(-13), toInt64(-2)), floor(toInt8(-13), toInt64(-2)); +SELECT round(toInt8(-13), toFloat32(-2.1)), ceil(toInt8(-13), toFloat32(-2.1)), floor(toInt8(-13), toFloat32(-2.1)); +SELECT round(toInt8(-13), toFloat64(-2.1)), ceil(toInt8(-13), toFloat64(-2.1)), floor(toInt8(-13), toFloat64(-2.1)); + +/* Int16 */ + +SELECT round(toInt16(-13), toUInt8(2)), ceil(toInt16(-13), toUInt8(2)), floor(toInt16(-13), toUInt8(2)); +SELECT round(toInt16(-13), toUInt16(2)), ceil(toInt16(-13), toUInt16(2)), floor(toInt16(-13), toUInt16(2)); +SELECT round(toInt16(-13), toUInt32(2)), ceil(toInt16(-13), toUInt32(2)), floor(toInt16(-13), toUInt32(2)); +SELECT round(toInt16(-13), toUInt64(2)), ceil(toInt16(-13), toUInt64(2)), floor(toInt16(-13), toUInt64(2)); +SELECT round(toInt16(-13), toInt8(2)), ceil(toInt16(-13), toInt8(2)), floor(toInt16(-13), toInt8(2)); +SELECT round(toInt16(-13), toInt16(2)), ceil(toInt16(-13), toInt16(2)), floor(toInt16(-13), toInt16(2)); +SELECT round(toInt16(-13), toInt32(2)), ceil(toInt16(-13), toInt32(2)), floor(toInt16(-13), toInt32(2)); +SELECT round(toInt16(-13), toInt64(2)), ceil(toInt16(-13), toInt64(2)), floor(toInt16(-13), toInt64(2)); +SELECT round(toInt16(-13), toFloat32(2.1)), ceil(toInt16(-13), toFloat32(2.1)), floor(toInt16(-13), toFloat32(2.1)); +SELECT round(toInt16(-13), toFloat64(2.1)), ceil(toInt16(-13), toFloat64(2.1)), floor(toInt16(-13), toFloat64(2.1)); + +SELECT round(toInt16(-13), toUInt8(1)), ceil(toInt16(-13), toUInt8(1)), floor(toInt16(-13), toUInt8(1)); +SELECT round(toInt16(-13), toUInt16(1)), ceil(toInt16(-13), toUInt16(1)), floor(toInt16(-13), toUInt16(1)); +SELECT round(toInt16(-13), toUInt32(1)), ceil(toInt16(-13), toUInt32(1)), floor(toInt16(-13), toUInt32(1)); +SELECT round(toInt16(-13), toUInt64(1)), ceil(toInt16(-13), toUInt64(1)), floor(toInt16(-13), toUInt64(1)); +SELECT round(toInt16(-13), toInt8(1)), ceil(toInt16(-13), toInt8(1)), floor(toInt16(-13), toInt8(1)); +SELECT round(toInt16(-13), toInt16(1)), ceil(toInt16(-13), toInt16(1)), floor(toInt16(-13), toInt16(1)); +SELECT round(toInt16(-13), toInt32(1)), ceil(toInt16(-13), toInt32(1)), floor(toInt16(-13), toInt32(1)); +SELECT round(toInt16(-13), toInt64(1)), ceil(toInt16(-13), toInt64(1)), floor(toInt16(-13), toInt64(1)); +SELECT round(toInt16(-13), toFloat32(1.1)), ceil(toInt16(-13), toFloat32(1.1)), floor(toInt16(-13), toFloat32(1.1)); +SELECT round(toInt16(-13), toFloat64(1.1)), ceil(toInt16(-13), toFloat64(1.1)), floor(toInt16(-13), toFloat64(1.1)); + +SELECT round(toInt16(-13), toUInt16(0)), ceil(toInt16(-13), toUInt16(0)), floor(toInt16(-13), toUInt16(0)); +SELECT round(toInt16(-13), toUInt32(0)), ceil(toInt16(-13), toUInt32(0)), floor(toInt16(-13), toUInt32(0)); +SELECT round(toInt16(-13), toUInt64(0)), ceil(toInt16(-13), toUInt64(0)), floor(toInt16(-13), toUInt64(0)); +SELECT round(toInt16(-13), toInt8(0)), ceil(toInt16(-13), toInt8(0)), floor(toInt16(-13), toInt8(0)); +SELECT round(toInt16(-13), toInt16(0)), ceil(toInt16(-13), toInt16(0)), floor(toInt16(-13), toInt16(0)); +SELECT round(toInt16(-13), toInt32(0)), ceil(toInt16(-13), toInt32(0)), floor(toInt16(-13), toInt32(0)); +SELECT round(toInt16(-13), toInt64(0)), ceil(toInt16(-13), toInt64(0)), floor(toInt16(-13), toInt64(0)); +SELECT round(toInt16(-13), toFloat32(0.1)), ceil(toInt16(-13), toFloat32(0.1)), floor(toInt16(-13), toFloat32(0.1)); +SELECT round(toInt16(-13), toFloat64(0.1)), ceil(toInt16(-13), toFloat64(0.1)), floor(toInt16(-13), toFloat64(0.1)); + +SELECT round(toInt16(-13), toInt8(-1)), ceil(toInt16(-13), toInt8(-1)), floor(toInt16(-13), toInt8(-1)); +SELECT round(toInt16(-13), toInt16(-1)), ceil(toInt16(-13), toInt16(-1)), floor(toInt16(-13), toInt16(-1)); +SELECT round(toInt16(-13), toInt32(-1)), ceil(toInt16(-13), toInt32(-1)), floor(toInt16(-13), toInt32(-1)); +SELECT round(toInt16(-13), toInt64(-1)), ceil(toInt16(-13), toInt64(-1)), floor(toInt16(-13), toInt64(-1)); +SELECT round(toInt16(-13), toFloat32(1.1)), ceil(toInt16(-13), toFloat32(-1.1)), floor(toInt16(-13), toFloat32(-1.1)); +SELECT round(toInt16(-13), toFloat64(1.1)), ceil(toInt16(-13), toFloat64(-1.1)), floor(toInt16(-13), toFloat64(-1.1)); + +SELECT round(toInt16(-13), toInt8(-2)), ceil(toInt16(-13), toInt8(-2)), floor(toInt16(-13), toInt8(-2)); +SELECT round(toInt16(-13), toInt16(-2)), ceil(toInt16(-13), toInt16(-2)), floor(toInt16(-13), toInt16(-2)); +SELECT round(toInt16(-13), toInt32(-2)), ceil(toInt16(-13), toInt32(-2)), floor(toInt16(-13), toInt32(-2)); +SELECT round(toInt16(-13), toInt64(-2)), ceil(toInt16(-13), toInt64(-2)), floor(toInt16(-13), toInt64(-2)); +SELECT round(toInt16(-13), toFloat32(-2.1)), ceil(toInt16(-13), toFloat32(-2.1)), floor(toInt16(-13), toFloat32(-2.1)); +SELECT round(toInt16(-13), toFloat64(-2.1)), ceil(toInt16(-13), toFloat64(-2.1)), floor(toInt16(-13), toFloat64(-2.1)); + +/* Int32 */ + +SELECT round(toInt32(-13), toUInt8(2)), ceil(toInt32(-13), toUInt8(2)), floor(toInt32(-13), toUInt8(2)); +SELECT round(toInt32(-13), toUInt16(2)), ceil(toInt32(-13), toUInt16(2)), floor(toInt32(-13), toUInt16(2)); +SELECT round(toInt32(-13), toUInt32(2)), ceil(toInt32(-13), toUInt32(2)), floor(toInt32(-13), toUInt32(2)); +SELECT round(toInt32(-13), toUInt64(2)), ceil(toInt32(-13), toUInt64(2)), floor(toInt32(-13), toUInt64(2)); +SELECT round(toInt32(-13), toInt8(2)), ceil(toInt32(-13), toInt8(2)), floor(toInt32(-13), toInt8(2)); +SELECT round(toInt32(-13), toInt16(2)), ceil(toInt32(-13), toInt16(2)), floor(toInt32(-13), toInt16(2)); +SELECT round(toInt32(-13), toInt32(2)), ceil(toInt32(-13), toInt32(2)), floor(toInt32(-13), toInt32(2)); +SELECT round(toInt32(-13), toInt64(2)), ceil(toInt32(-13), toInt64(2)), floor(toInt32(-13), toInt64(2)); +SELECT round(toInt32(-13), toFloat32(2.1)), ceil(toInt32(-13), toFloat32(2.1)), floor(toInt32(-13), toFloat32(2.1)); +SELECT round(toInt32(-13), toFloat64(2.1)), ceil(toInt32(-13), toFloat64(2.1)), floor(toInt32(-13), toFloat64(2.1)); + +SELECT round(toInt32(-13), toUInt8(1)), ceil(toInt32(-13), toUInt8(1)), floor(toInt32(-13), toUInt8(1)); +SELECT round(toInt32(-13), toUInt16(1)), ceil(toInt32(-13), toUInt16(1)), floor(toInt32(-13), toUInt16(1)); +SELECT round(toInt32(-13), toUInt32(1)), ceil(toInt32(-13), toUInt32(1)), floor(toInt32(-13), toUInt32(1)); +SELECT round(toInt32(-13), toUInt64(1)), ceil(toInt32(-13), toUInt64(1)), floor(toInt32(-13), toUInt64(1)); +SELECT round(toInt32(-13), toInt8(1)), ceil(toInt32(-13), toInt8(1)), floor(toInt32(-13), toInt8(1)); +SELECT round(toInt32(-13), toInt16(1)), ceil(toInt32(-13), toInt16(1)), floor(toInt32(-13), toInt16(1)); +SELECT round(toInt32(-13), toInt32(1)), ceil(toInt32(-13), toInt32(1)), floor(toInt32(-13), toInt32(1)); +SELECT round(toInt32(-13), toInt64(1)), ceil(toInt32(-13), toInt64(1)), floor(toInt32(-13), toInt64(1)); +SELECT round(toInt32(-13), toFloat32(1.1)), ceil(toInt32(-13), toFloat32(1.1)), floor(toInt32(-13), toFloat32(1.1)); +SELECT round(toInt32(-13), toFloat64(1.1)), ceil(toInt32(-13), toFloat64(1.1)), floor(toInt32(-13), toFloat64(1.1)); + +SELECT round(toInt32(-13), toUInt16(0)), ceil(toInt32(-13), toUInt16(0)), floor(toInt32(-13), toUInt16(0)); +SELECT round(toInt32(-13), toUInt32(0)), ceil(toInt32(-13), toUInt32(0)), floor(toInt32(-13), toUInt32(0)); +SELECT round(toInt32(-13), toUInt64(0)), ceil(toInt32(-13), toUInt64(0)), floor(toInt32(-13), toUInt64(0)); +SELECT round(toInt32(-13), toInt8(0)), ceil(toInt32(-13), toInt8(0)), floor(toInt32(-13), toInt8(0)); +SELECT round(toInt32(-13), toInt16(0)), ceil(toInt32(-13), toInt16(0)), floor(toInt32(-13), toInt16(0)); +SELECT round(toInt32(-13), toInt32(0)), ceil(toInt32(-13), toInt32(0)), floor(toInt32(-13), toInt32(0)); +SELECT round(toInt32(-13), toInt64(0)), ceil(toInt32(-13), toInt64(0)), floor(toInt32(-13), toInt64(0)); +SELECT round(toInt32(-13), toFloat32(0.1)), ceil(toInt32(-13), toFloat32(0.1)), floor(toInt32(-13), toFloat32(0.1)); +SELECT round(toInt32(-13), toFloat64(0.1)), ceil(toInt32(-13), toFloat64(0.1)), floor(toInt32(-13), toFloat64(0.1)); + +SELECT round(toInt32(-13), toInt8(-1)), ceil(toInt32(-13), toInt8(-1)), floor(toInt32(-13), toInt8(-1)); +SELECT round(toInt32(-13), toInt16(-1)), ceil(toInt32(-13), toInt16(-1)), floor(toInt32(-13), toInt16(-1)); +SELECT round(toInt32(-13), toInt32(-1)), ceil(toInt32(-13), toInt32(-1)), floor(toInt32(-13), toInt32(-1)); +SELECT round(toInt32(-13), toInt64(-1)), ceil(toInt32(-13), toInt64(-1)), floor(toInt32(-13), toInt64(-1)); +SELECT round(toInt32(-13), toFloat32(1.1)), ceil(toInt32(-13), toFloat32(-1.1)), floor(toInt32(-13), toFloat32(-1.1)); +SELECT round(toInt32(-13), toFloat64(1.1)), ceil(toInt32(-13), toFloat64(-1.1)), floor(toInt32(-13), toFloat64(-1.1)); + +SELECT round(toInt32(-13), toInt8(-2)), ceil(toInt32(-13), toInt8(-2)), floor(toInt32(-13), toInt8(-2)); +SELECT round(toInt32(-13), toInt16(-2)), ceil(toInt32(-13), toInt16(-2)), floor(toInt32(-13), toInt16(-2)); +SELECT round(toInt32(-13), toInt32(-2)), ceil(toInt32(-13), toInt32(-2)), floor(toInt32(-13), toInt32(-2)); +SELECT round(toInt32(-13), toInt64(-2)), ceil(toInt32(-13), toInt64(-2)), floor(toInt32(-13), toInt64(-2)); +SELECT round(toInt32(-13), toFloat32(-2.1)), ceil(toInt32(-13), toFloat32(-2.1)), floor(toInt32(-13), toFloat32(-2.1)); +SELECT round(toInt32(-13), toFloat64(-2.1)), ceil(toInt32(-13), toFloat64(-2.1)), floor(toInt32(-13), toFloat64(-2.1)); + +/* Int64 */ + +SELECT round(toInt64(-13), toUInt8(2)), ceil(toInt64(-13), toUInt8(2)), floor(toInt64(-13), toUInt8(2)); +SELECT round(toInt64(-13), toUInt16(2)), ceil(toInt64(-13), toUInt16(2)), floor(toInt64(-13), toUInt16(2)); +SELECT round(toInt64(-13), toUInt32(2)), ceil(toInt64(-13), toUInt32(2)), floor(toInt64(-13), toUInt32(2)); +SELECT round(toInt64(-13), toUInt64(2)), ceil(toInt64(-13), toUInt64(2)), floor(toInt64(-13), toUInt64(2)); +SELECT round(toInt64(-13), toInt8(2)), ceil(toInt64(-13), toInt8(2)), floor(toInt64(-13), toInt8(2)); +SELECT round(toInt64(-13), toInt16(2)), ceil(toInt64(-13), toInt16(2)), floor(toInt64(-13), toInt16(2)); +SELECT round(toInt64(-13), toInt32(2)), ceil(toInt64(-13), toInt32(2)), floor(toInt64(-13), toInt32(2)); +SELECT round(toInt64(-13), toInt64(2)), ceil(toInt64(-13), toInt64(2)), floor(toInt64(-13), toInt64(2)); +SELECT round(toInt64(-13), toFloat32(2.1)), ceil(toInt64(-13), toFloat32(2.1)), floor(toInt64(-13), toFloat32(2.1)); +SELECT round(toInt64(-13), toFloat64(2.1)), ceil(toInt64(-13), toFloat64(2.1)), floor(toInt64(-13), toFloat64(2.1)); + +SELECT round(toInt64(-13), toUInt8(1)), ceil(toInt64(-13), toUInt8(1)), floor(toInt64(-13), toUInt8(1)); +SELECT round(toInt64(-13), toUInt16(1)), ceil(toInt64(-13), toUInt16(1)), floor(toInt64(-13), toUInt16(1)); +SELECT round(toInt64(-13), toUInt32(1)), ceil(toInt64(-13), toUInt32(1)), floor(toInt64(-13), toUInt32(1)); +SELECT round(toInt64(-13), toUInt64(1)), ceil(toInt64(-13), toUInt64(1)), floor(toInt64(-13), toUInt64(1)); +SELECT round(toInt64(-13), toInt8(1)), ceil(toInt64(-13), toInt8(1)), floor(toInt64(-13), toInt8(1)); +SELECT round(toInt64(-13), toInt16(1)), ceil(toInt64(-13), toInt16(1)), floor(toInt64(-13), toInt16(1)); +SELECT round(toInt64(-13), toInt32(1)), ceil(toInt64(-13), toInt32(1)), floor(toInt64(-13), toInt32(1)); +SELECT round(toInt64(-13), toInt64(1)), ceil(toInt64(-13), toInt64(1)), floor(toInt64(-13), toInt64(1)); +SELECT round(toInt64(-13), toFloat32(1.1)), ceil(toInt64(-13), toFloat32(1.1)), floor(toInt64(-13), toFloat32(1.1)); +SELECT round(toInt64(-13), toFloat64(1.1)), ceil(toInt64(-13), toFloat64(1.1)), floor(toInt64(-13), toFloat64(1.1)); + +SELECT round(toInt64(-13), toUInt16(0)), ceil(toInt64(-13), toUInt16(0)), floor(toInt64(-13), toUInt16(0)); +SELECT round(toInt64(-13), toUInt32(0)), ceil(toInt64(-13), toUInt32(0)), floor(toInt64(-13), toUInt32(0)); +SELECT round(toInt64(-13), toUInt64(0)), ceil(toInt64(-13), toUInt64(0)), floor(toInt64(-13), toUInt64(0)); +SELECT round(toInt64(-13), toInt8(0)), ceil(toInt64(-13), toInt8(0)), floor(toInt64(-13), toInt8(0)); +SELECT round(toInt64(-13), toInt16(0)), ceil(toInt64(-13), toInt16(0)), floor(toInt64(-13), toInt16(0)); +SELECT round(toInt64(-13), toInt32(0)), ceil(toInt64(-13), toInt32(0)), floor(toInt64(-13), toInt32(0)); +SELECT round(toInt64(-13), toInt64(0)), ceil(toInt64(-13), toInt64(0)), floor(toInt64(-13), toInt64(0)); +SELECT round(toInt64(-13), toFloat32(0.1)), ceil(toInt64(-13), toFloat32(0.1)), floor(toInt64(-13), toFloat32(0.1)); +SELECT round(toInt64(-13), toFloat64(0.1)), ceil(toInt64(-13), toFloat64(0.1)), floor(toInt64(-13), toFloat64(0.1)); + +SELECT round(toInt64(-13), toInt8(-1)), ceil(toInt64(-13), toInt8(-1)), floor(toInt64(-13), toInt8(-1)); +SELECT round(toInt64(-13), toInt16(-1)), ceil(toInt64(-13), toInt16(-1)), floor(toInt64(-13), toInt16(-1)); +SELECT round(toInt64(-13), toInt32(-1)), ceil(toInt64(-13), toInt32(-1)), floor(toInt64(-13), toInt32(-1)); +SELECT round(toInt64(-13), toInt64(-1)), ceil(toInt64(-13), toInt64(-1)), floor(toInt64(-13), toInt64(-1)); +SELECT round(toInt64(-13), toFloat32(1.1)), ceil(toInt64(-13), toFloat32(-1.1)), floor(toInt64(-13), toFloat32(-1.1)); +SELECT round(toInt64(-13), toFloat64(1.1)), ceil(toInt64(-13), toFloat64(-1.1)), floor(toInt64(-13), toFloat64(-1.1)); + +SELECT round(toInt64(-13), toInt8(-2)), ceil(toInt64(-13), toInt8(-2)), floor(toInt64(-13), toInt8(-2)); +SELECT round(toInt64(-13), toInt16(-2)), ceil(toInt64(-13), toInt16(-2)), floor(toInt64(-13), toInt16(-2)); +SELECT round(toInt64(-13), toInt32(-2)), ceil(toInt64(-13), toInt32(-2)), floor(toInt64(-13), toInt32(-2)); +SELECT round(toInt64(-13), toInt64(-2)), ceil(toInt64(-13), toInt64(-2)), floor(toInt64(-13), toInt64(-2)); +SELECT round(toInt64(-13), toFloat32(-2.1)), ceil(toInt64(-13), toFloat32(-2.1)), floor(toInt64(-13), toFloat32(-2.1)); +SELECT round(toInt64(-13), toFloat64(-2.1)), ceil(toInt64(-13), toFloat64(-2.1)), floor(toInt64(-13), toFloat64(-2.1)); + +/* Float32 */ + +SELECT round(toFloat32(-13), toUInt8(2)), ceil(toFloat32(-13), toUInt8(2)), floor(toFloat32(-13), toUInt8(2)); +SELECT round(toFloat32(-13), toUInt16(2)), ceil(toFloat32(-13), toUInt16(2)), floor(toFloat32(-13), toUInt16(2)); +SELECT round(toFloat32(-13), toUInt32(2)), ceil(toFloat32(-13), toUInt32(2)), floor(toFloat32(-13), toUInt32(2)); +SELECT round(toFloat32(-13), toUInt64(2)), ceil(toFloat32(-13), toUInt64(2)), floor(toFloat32(-13), toUInt64(2)); +SELECT round(toFloat32(-13), toInt8(2)), ceil(toFloat32(-13), toInt8(2)), floor(toFloat32(-13), toInt8(2)); +SELECT round(toFloat32(-13), toInt16(2)), ceil(toFloat32(-13), toInt16(2)), floor(toFloat32(-13), toInt16(2)); +SELECT round(toFloat32(-13), toInt32(2)), ceil(toFloat32(-13), toInt32(2)), floor(toFloat32(-13), toInt32(2)); +SELECT round(toFloat32(-13), toInt64(2)), ceil(toFloat32(-13), toInt64(2)), floor(toFloat32(-13), toInt64(2)); +SELECT round(toFloat32(-13), toFloat32(2.1)), ceil(toFloat32(-13), toFloat32(2.1)), floor(toFloat32(-13), toFloat32(2.1)); +SELECT round(toFloat32(-13), toFloat64(2.1)), ceil(toFloat32(-13), toFloat64(2.1)), floor(toFloat32(-13), toFloat64(2.1)); + +SELECT round(toFloat32(-13), toUInt8(1)), ceil(toFloat32(-13), toUInt8(1)), floor(toFloat32(-13), toUInt8(1)); +SELECT round(toFloat32(-13), toUInt16(1)), ceil(toFloat32(-13), toUInt16(1)), floor(toFloat32(-13), toUInt16(1)); +SELECT round(toFloat32(-13), toUInt32(1)), ceil(toFloat32(-13), toUInt32(1)), floor(toFloat32(-13), toUInt32(1)); +SELECT round(toFloat32(-13), toUInt64(1)), ceil(toFloat32(-13), toUInt64(1)), floor(toFloat32(-13), toUInt64(1)); +SELECT round(toFloat32(-13), toInt8(1)), ceil(toFloat32(-13), toInt8(1)), floor(toFloat32(-13), toInt8(1)); +SELECT round(toFloat32(-13), toInt16(1)), ceil(toFloat32(-13), toInt16(1)), floor(toFloat32(-13), toInt16(1)); +SELECT round(toFloat32(-13), toInt32(1)), ceil(toFloat32(-13), toInt32(1)), floor(toFloat32(-13), toInt32(1)); +SELECT round(toFloat32(-13), toInt64(1)), ceil(toFloat32(-13), toInt64(1)), floor(toFloat32(-13), toInt64(1)); +SELECT round(toFloat32(-13), toFloat32(1.1)), ceil(toFloat32(-13), toFloat32(1.1)), floor(toFloat32(-13), toFloat32(1.1)); +SELECT round(toFloat32(-13), toFloat64(1.1)), ceil(toFloat32(-13), toFloat64(1.1)), floor(toFloat32(-13), toFloat64(1.1)); + +SELECT round(toFloat32(-13), toUInt16(0)), ceil(toFloat32(-13), toUInt16(0)), floor(toFloat32(-13), toUInt16(0)); +SELECT round(toFloat32(-13), toUInt32(0)), ceil(toFloat32(-13), toUInt32(0)), floor(toFloat32(-13), toUInt32(0)); +SELECT round(toFloat32(-13), toUInt64(0)), ceil(toFloat32(-13), toUInt64(0)), floor(toFloat32(-13), toUInt64(0)); +SELECT round(toFloat32(-13), toInt8(0)), ceil(toFloat32(-13), toInt8(0)), floor(toFloat32(-13), toInt8(0)); +SELECT round(toFloat32(-13), toInt16(0)), ceil(toFloat32(-13), toInt16(0)), floor(toFloat32(-13), toInt16(0)); +SELECT round(toFloat32(-13), toInt32(0)), ceil(toFloat32(-13), toInt32(0)), floor(toFloat32(-13), toInt32(0)); +SELECT round(toFloat32(-13), toInt64(0)), ceil(toFloat32(-13), toInt64(0)), floor(toFloat32(-13), toInt64(0)); +SELECT round(toFloat32(-13), toFloat32(0.1)), ceil(toFloat32(-13), toFloat32(0.1)), floor(toFloat32(-13), toFloat32(0.1)); +SELECT round(toFloat32(-13), toFloat64(0.1)), ceil(toFloat32(-13), toFloat64(0.1)), floor(toFloat32(-13), toFloat64(0.1)); + +SELECT round(toFloat32(-13), toInt8(-1)), ceil(toFloat32(-13), toInt8(-1)), floor(toFloat32(-13), toInt8(-1)); +SELECT round(toFloat32(-13), toInt16(-1)), ceil(toFloat32(-13), toInt16(-1)), floor(toFloat32(-13), toInt16(-1)); +SELECT round(toFloat32(-13), toInt32(-1)), ceil(toFloat32(-13), toInt32(-1)), floor(toFloat32(-13), toInt32(-1)); +SELECT round(toFloat32(-13), toInt64(-1)), ceil(toFloat32(-13), toInt64(-1)), floor(toFloat32(-13), toInt64(-1)); +SELECT round(toFloat32(-13), toFloat32(1.1)), ceil(toFloat32(-13), toFloat32(-1.1)), floor(toFloat32(-13), toFloat32(-1.1)); +SELECT round(toFloat32(-13), toFloat64(1.1)), ceil(toFloat32(-13), toFloat64(-1.1)), floor(toFloat32(-13), toFloat64(-1.1)); + +SELECT round(toFloat32(-13), toInt8(-2)), ceil(toFloat32(-13), toInt8(-2)), floor(toFloat32(-13), toInt8(-2)); +SELECT round(toFloat32(-13), toInt16(-2)), ceil(toFloat32(-13), toInt16(-2)), floor(toFloat32(-13), toInt16(-2)); +SELECT round(toFloat32(-13), toInt32(-2)), ceil(toFloat32(-13), toInt32(-2)), floor(toFloat32(-13), toInt32(-2)); +SELECT round(toFloat32(-13), toInt64(-2)), ceil(toFloat32(-13), toInt64(-2)), floor(toFloat32(-13), toInt64(-2)); +SELECT round(toFloat32(-13), toFloat32(-2.1)), ceil(toFloat32(-13), toFloat32(-2.1)), floor(toFloat32(-13), toFloat32(-2.1)); +SELECT round(toFloat32(-13), toFloat64(-2.1)), ceil(toFloat32(-13), toFloat64(-2.1)), floor(toFloat32(-13), toFloat64(-2.1)); + +/* Float64 */ + +SELECT round(toFloat64(-13), toUInt8(2)), ceil(toFloat64(-13), toUInt8(2)), floor(toFloat64(-13), toUInt8(2)); +SELECT round(toFloat64(-13), toUInt16(2)), ceil(toFloat64(-13), toUInt16(2)), floor(toFloat64(-13), toUInt16(2)); +SELECT round(toFloat64(-13), toUInt32(2)), ceil(toFloat64(-13), toUInt32(2)), floor(toFloat64(-13), toUInt32(2)); +SELECT round(toFloat64(-13), toUInt64(2)), ceil(toFloat64(-13), toUInt64(2)), floor(toFloat64(-13), toUInt64(2)); +SELECT round(toFloat64(-13), toInt8(2)), ceil(toFloat64(-13), toInt8(2)), floor(toFloat64(-13), toInt8(2)); +SELECT round(toFloat64(-13), toInt16(2)), ceil(toFloat64(-13), toInt16(2)), floor(toFloat64(-13), toInt16(2)); +SELECT round(toFloat64(-13), toInt32(2)), ceil(toFloat64(-13), toInt32(2)), floor(toFloat64(-13), toInt32(2)); +SELECT round(toFloat64(-13), toInt64(2)), ceil(toFloat64(-13), toInt64(2)), floor(toFloat64(-13), toInt64(2)); +SELECT round(toFloat64(-13), toFloat32(2.1)), ceil(toFloat64(-13), toFloat32(2.1)), floor(toFloat64(-13), toFloat32(2.1)); +SELECT round(toFloat64(-13), toFloat64(2.1)), ceil(toFloat64(-13), toFloat64(2.1)), floor(toFloat64(-13), toFloat64(2.1)); + +SELECT round(toFloat64(-13), toUInt8(1)), ceil(toFloat64(-13), toUInt8(1)), floor(toFloat64(-13), toUInt8(1)); +SELECT round(toFloat64(-13), toUInt16(1)), ceil(toFloat64(-13), toUInt16(1)), floor(toFloat64(-13), toUInt16(1)); +SELECT round(toFloat64(-13), toUInt32(1)), ceil(toFloat64(-13), toUInt32(1)), floor(toFloat64(-13), toUInt32(1)); +SELECT round(toFloat64(-13), toUInt64(1)), ceil(toFloat64(-13), toUInt64(1)), floor(toFloat64(-13), toUInt64(1)); +SELECT round(toFloat64(-13), toInt8(1)), ceil(toFloat64(-13), toInt8(1)), floor(toFloat64(-13), toInt8(1)); +SELECT round(toFloat64(-13), toInt16(1)), ceil(toFloat64(-13), toInt16(1)), floor(toFloat64(-13), toInt16(1)); +SELECT round(toFloat64(-13), toInt32(1)), ceil(toFloat64(-13), toInt32(1)), floor(toFloat64(-13), toInt32(1)); +SELECT round(toFloat64(-13), toInt64(1)), ceil(toFloat64(-13), toInt64(1)), floor(toFloat64(-13), toInt64(1)); +SELECT round(toFloat64(-13), toFloat32(1.1)), ceil(toFloat64(-13), toFloat32(1.1)), floor(toFloat64(-13), toFloat32(1.1)); +SELECT round(toFloat64(-13), toFloat64(1.1)), ceil(toFloat64(-13), toFloat64(1.1)), floor(toFloat64(-13), toFloat64(1.1)); + +SELECT round(toFloat64(-13), toUInt16(0)), ceil(toFloat64(-13), toUInt16(0)), floor(toFloat64(-13), toUInt16(0)); +SELECT round(toFloat64(-13), toUInt32(0)), ceil(toFloat64(-13), toUInt32(0)), floor(toFloat64(-13), toUInt32(0)); +SELECT round(toFloat64(-13), toUInt64(0)), ceil(toFloat64(-13), toUInt64(0)), floor(toFloat64(-13), toUInt64(0)); +SELECT round(toFloat64(-13), toInt8(0)), ceil(toFloat64(-13), toInt8(0)), floor(toFloat64(-13), toInt8(0)); +SELECT round(toFloat64(-13), toInt16(0)), ceil(toFloat64(-13), toInt16(0)), floor(toFloat64(-13), toInt16(0)); +SELECT round(toFloat64(-13), toInt32(0)), ceil(toFloat64(-13), toInt32(0)), floor(toFloat64(-13), toInt32(0)); +SELECT round(toFloat64(-13), toInt64(0)), ceil(toFloat64(-13), toInt64(0)), floor(toFloat64(-13), toInt64(0)); +SELECT round(toFloat64(-13), toFloat32(0.1)), ceil(toFloat64(-13), toFloat32(0.1)), floor(toFloat64(-13), toFloat32(0.1)); +SELECT round(toFloat64(-13), toFloat64(0.1)), ceil(toFloat64(-13), toFloat64(0.1)), floor(toFloat64(-13), toFloat64(0.1)); + +SELECT round(toFloat64(-13), toInt8(-1)), ceil(toFloat64(-13), toInt8(-1)), floor(toFloat64(-13), toInt8(-1)); +SELECT round(toFloat64(-13), toInt16(-1)), ceil(toFloat64(-13), toInt16(-1)), floor(toFloat64(-13), toInt16(-1)); +SELECT round(toFloat64(-13), toInt32(-1)), ceil(toFloat64(-13), toInt32(-1)), floor(toFloat64(-13), toInt32(-1)); +SELECT round(toFloat64(-13), toInt64(-1)), ceil(toFloat64(-13), toInt64(-1)), floor(toFloat64(-13), toInt64(-1)); +SELECT round(toFloat64(-13), toFloat32(1.1)), ceil(toFloat64(-13), toFloat32(-1.1)), floor(toFloat64(-13), toFloat32(-1.1)); +SELECT round(toFloat64(-13), toFloat64(1.1)), ceil(toFloat64(-13), toFloat64(-1.1)), floor(toFloat64(-13), toFloat64(-1.1)); + +SELECT round(toFloat64(-13), toInt8(-2)), ceil(toFloat64(-13), toInt8(-2)), floor(toFloat64(-13), toInt8(-2)); +SELECT round(toFloat64(-13), toInt16(-2)), ceil(toFloat64(-13), toInt16(-2)), floor(toFloat64(-13), toInt16(-2)); +SELECT round(toFloat64(-13), toInt32(-2)), ceil(toFloat64(-13), toInt32(-2)), floor(toFloat64(-13), toInt32(-2)); +SELECT round(toFloat64(-13), toInt64(-2)), ceil(toFloat64(-13), toInt64(-2)), floor(toFloat64(-13), toInt64(-2)); +SELECT round(toFloat64(-13), toFloat32(-2.1)), ceil(toFloat64(-13), toFloat32(-2.1)), floor(toFloat64(-13), toFloat32(-2.1)); +SELECT round(toFloat64(-13), toFloat64(-2.1)), ceil(toFloat64(-13), toFloat64(-2.1)), floor(toFloat64(-13), toFloat64(-2.1)); + +/* Положительное число с плавающей точкой */ + +SELECT round(toFloat64(2.718281828459), toUInt8(2)), ceil(toFloat64(2.718281828459), toUInt8(2)), floor(toFloat64(2.718281828459), toUInt8(2)); +SELECT round(toFloat64(2.718281828459), toUInt16(2)), ceil(toFloat64(2.718281828459), toUInt16(2)), floor(toFloat64(2.718281828459), toUInt16(2)); +SELECT round(toFloat64(2.718281828459), toUInt32(2)), ceil(toFloat64(2.718281828459), toUInt32(2)), floor(toFloat64(2.718281828459), toUInt32(2)); +SELECT round(toFloat64(2.718281828459), toUInt64(2)), ceil(toFloat64(2.718281828459), toUInt64(2)), floor(toFloat64(2.718281828459), toUInt64(2)); +SELECT round(toFloat64(2.718281828459), toInt8(2)), ceil(toFloat64(2.718281828459), toInt8(2)), floor(toFloat64(2.718281828459), toInt8(2)); +SELECT round(toFloat64(2.718281828459), toInt16(2)), ceil(toFloat64(2.718281828459), toInt16(2)), floor(toFloat64(2.718281828459), toInt16(2)); +SELECT round(toFloat64(2.718281828459), toInt32(2)), ceil(toFloat64(2.718281828459), toInt32(2)), floor(toFloat64(2.718281828459), toInt32(2)); +SELECT round(toFloat64(2.718281828459), toInt64(2)), ceil(toFloat64(2.718281828459), toInt64(2)), floor(toFloat64(2.718281828459), toInt64(2)); +SELECT round(toFloat64(2.718281828459), toFloat32(2.1)), ceil(toFloat64(2.718281828459), toFloat32(2.1)), floor(toFloat64(2.718281828459), toFloat32(2.1)); +SELECT round(toFloat64(2.718281828459), toFloat64(2.1)), ceil(toFloat64(2.718281828459), toFloat64(2.1)), floor(toFloat64(2.718281828459), toFloat64(2.1)); + +SELECT round(toFloat64(2.718281828459), toUInt8(1)), ceil(toFloat64(2.718281828459), toUInt8(1)), floor(toFloat64(2.718281828459), toUInt8(1)); +SELECT round(toFloat64(2.718281828459), toUInt16(1)), ceil(toFloat64(2.718281828459), toUInt16(1)), floor(toFloat64(2.718281828459), toUInt16(1)); +SELECT round(toFloat64(2.718281828459), toUInt32(1)), ceil(toFloat64(2.718281828459), toUInt32(1)), floor(toFloat64(2.718281828459), toUInt32(1)); +SELECT round(toFloat64(2.718281828459), toUInt64(1)), ceil(toFloat64(2.718281828459), toUInt64(1)), floor(toFloat64(2.718281828459), toUInt64(1)); +SELECT round(toFloat64(2.718281828459), toInt8(1)), ceil(toFloat64(2.718281828459), toInt8(1)), floor(toFloat64(2.718281828459), toInt8(1)); +SELECT round(toFloat64(2.718281828459), toInt16(1)), ceil(toFloat64(2.718281828459), toInt16(1)), floor(toFloat64(2.718281828459), toInt16(1)); +SELECT round(toFloat64(2.718281828459), toInt32(1)), ceil(toFloat64(2.718281828459), toInt32(1)), floor(toFloat64(2.718281828459), toInt32(1)); +SELECT round(toFloat64(2.718281828459), toInt64(1)), ceil(toFloat64(2.718281828459), toInt64(1)), floor(toFloat64(2.718281828459), toInt64(1)); +SELECT round(toFloat64(2.718281828459), toFloat32(1.1)), ceil(toFloat64(2.718281828459), toFloat32(1.1)), floor(toFloat64(2.718281828459), toFloat32(1.1)); +SELECT round(toFloat64(2.718281828459), toFloat64(1.1)), ceil(toFloat64(2.718281828459), toFloat64(1.1)), floor(toFloat64(2.718281828459), toFloat64(1.1)); + +SELECT round(toFloat64(2.718281828459), toUInt16(0)), ceil(toFloat64(2.718281828459), toUInt16(0)), floor(toFloat64(2.718281828459), toUInt16(0)); +SELECT round(toFloat64(2.718281828459), toUInt32(0)), ceil(toFloat64(2.718281828459), toUInt32(0)), floor(toFloat64(2.718281828459), toUInt32(0)); +SELECT round(toFloat64(2.718281828459), toUInt64(0)), ceil(toFloat64(2.718281828459), toUInt64(0)), floor(toFloat64(2.718281828459), toUInt64(0)); +SELECT round(toFloat64(2.718281828459), toInt8(0)), ceil(toFloat64(2.718281828459), toInt8(0)), floor(toFloat64(2.718281828459), toInt8(0)); +SELECT round(toFloat64(2.718281828459), toInt16(0)), ceil(toFloat64(2.718281828459), toInt16(0)), floor(toFloat64(2.718281828459), toInt16(0)); +SELECT round(toFloat64(2.718281828459), toInt32(0)), ceil(toFloat64(2.718281828459), toInt32(0)), floor(toFloat64(2.718281828459), toInt32(0)); +SELECT round(toFloat64(2.718281828459), toInt64(0)), ceil(toFloat64(2.718281828459), toInt64(0)), floor(toFloat64(2.718281828459), toInt64(0)); +SELECT round(toFloat64(2.718281828459), toFloat32(0.1)), ceil(toFloat64(2.718281828459), toFloat32(0.1)), floor(toFloat64(2.718281828459), toFloat32(0.1)); +SELECT round(toFloat64(2.718281828459), toFloat64(0.1)), ceil(toFloat64(2.718281828459), toFloat64(0.1)), floor(toFloat64(2.718281828459), toFloat64(0.1)); + +SELECT round(toFloat64(2.718281828459), toInt8(-1)), ceil(toFloat64(2.718281828459), toInt8(-1)), floor(toFloat64(2.718281828459), toInt8(-1)); +SELECT round(toFloat64(2.718281828459), toInt16(-1)), ceil(toFloat64(2.718281828459), toInt16(-1)), floor(toFloat64(2.718281828459), toInt16(-1)); +SELECT round(toFloat64(2.718281828459), toInt32(-1)), ceil(toFloat64(2.718281828459), toInt32(-1)), floor(toFloat64(2.718281828459), toInt32(-1)); +SELECT round(toFloat64(2.718281828459), toInt64(-1)), ceil(toFloat64(2.718281828459), toInt64(-1)), floor(toFloat64(2.718281828459), toInt64(-1)); +SELECT round(toFloat64(2.718281828459), toFloat32(1.1)), ceil(toFloat64(2.718281828459), toFloat32(-1.1)), floor(toFloat64(2.718281828459), toFloat32(-1.1)); +SELECT round(toFloat64(2.718281828459), toFloat64(1.1)), ceil(toFloat64(2.718281828459), toFloat64(-1.1)), floor(toFloat64(2.718281828459), toFloat64(-1.1)); + +SELECT round(toFloat64(2.718281828459), toInt8(-2)), ceil(toFloat64(2.718281828459), toInt8(-2)), floor(toFloat64(2.718281828459), toInt8(-2)); +SELECT round(toFloat64(2.718281828459), toInt16(-2)), ceil(toFloat64(2.718281828459), toInt16(-2)), floor(toFloat64(2.718281828459), toInt16(-2)); +SELECT round(toFloat64(2.718281828459), toInt32(-2)), ceil(toFloat64(2.718281828459), toInt32(-2)), floor(toFloat64(2.718281828459), toInt32(-2)); +SELECT round(toFloat64(2.718281828459), toInt64(-2)), ceil(toFloat64(2.718281828459), toInt64(-2)), floor(toFloat64(2.718281828459), toInt64(-2)); +SELECT round(toFloat64(2.718281828459), toFloat32(-2.1)), ceil(toFloat64(2.718281828459), toFloat32(-2.1)), floor(toFloat64(2.718281828459), toFloat32(-2.1)); +SELECT round(toFloat64(2.718281828459), toFloat64(-2.1)), ceil(toFloat64(2.718281828459), toFloat64(-2.1)), floor(toFloat64(2.718281828459), toFloat64(-2.1)); + +/* Отрицательное число с плавающей точкой */ + +SELECT round(toFloat64(-2.718281828459), toUInt8(2)), ceil(toFloat64(-2.718281828459), toUInt8(2)), floor(toFloat64(-2.718281828459), toUInt8(2)); +SELECT round(toFloat64(-2.718281828459), toUInt16(2)), ceil(toFloat64(-2.718281828459), toUInt16(2)), floor(toFloat64(-2.718281828459), toUInt16(2)); +SELECT round(toFloat64(-2.718281828459), toUInt32(2)), ceil(toFloat64(-2.718281828459), toUInt32(2)), floor(toFloat64(-2.718281828459), toUInt32(2)); +SELECT round(toFloat64(-2.718281828459), toUInt64(2)), ceil(toFloat64(-2.718281828459), toUInt64(2)), floor(toFloat64(-2.718281828459), toUInt64(2)); +SELECT round(toFloat64(-2.718281828459), toInt8(2)), ceil(toFloat64(-2.718281828459), toInt8(2)), floor(toFloat64(-2.718281828459), toInt8(2)); +SELECT round(toFloat64(-2.718281828459), toInt16(2)), ceil(toFloat64(-2.718281828459), toInt16(2)), floor(toFloat64(-2.718281828459), toInt16(2)); +SELECT round(toFloat64(-2.718281828459), toInt32(2)), ceil(toFloat64(-2.718281828459), toInt32(2)), floor(toFloat64(-2.718281828459), toInt32(2)); +SELECT round(toFloat64(-2.718281828459), toInt64(2)), ceil(toFloat64(-2.718281828459), toInt64(2)), floor(toFloat64(-2.718281828459), toInt64(2)); +SELECT round(toFloat64(-2.718281828459), toFloat32(2.1)), ceil(toFloat64(-2.718281828459), toFloat32(2.1)), floor(toFloat64(-2.718281828459), toFloat32(2.1)); +SELECT round(toFloat64(-2.718281828459), toFloat64(2.1)), ceil(toFloat64(-2.718281828459), toFloat64(2.1)), floor(toFloat64(-2.718281828459), toFloat64(2.1)); + +SELECT round(toFloat64(-2.718281828459), toUInt8(1)), ceil(toFloat64(-2.718281828459), toUInt8(1)), floor(toFloat64(-2.718281828459), toUInt8(1)); +SELECT round(toFloat64(-2.718281828459), toUInt16(1)), ceil(toFloat64(-2.718281828459), toUInt16(1)), floor(toFloat64(-2.718281828459), toUInt16(1)); +SELECT round(toFloat64(-2.718281828459), toUInt32(1)), ceil(toFloat64(-2.718281828459), toUInt32(1)), floor(toFloat64(-2.718281828459), toUInt32(1)); +SELECT round(toFloat64(-2.718281828459), toUInt64(1)), ceil(toFloat64(-2.718281828459), toUInt64(1)), floor(toFloat64(-2.718281828459), toUInt64(1)); +SELECT round(toFloat64(-2.718281828459), toInt8(1)), ceil(toFloat64(-2.718281828459), toInt8(1)), floor(toFloat64(-2.718281828459), toInt8(1)); +SELECT round(toFloat64(-2.718281828459), toInt16(1)), ceil(toFloat64(-2.718281828459), toInt16(1)), floor(toFloat64(-2.718281828459), toInt16(1)); +SELECT round(toFloat64(-2.718281828459), toInt32(1)), ceil(toFloat64(-2.718281828459), toInt32(1)), floor(toFloat64(-2.718281828459), toInt32(1)); +SELECT round(toFloat64(-2.718281828459), toInt64(1)), ceil(toFloat64(-2.718281828459), toInt64(1)), floor(toFloat64(-2.718281828459), toInt64(1)); +SELECT round(toFloat64(-2.718281828459), toFloat32(1.1)), ceil(toFloat64(-2.718281828459), toFloat32(1.1)), floor(toFloat64(-2.718281828459), toFloat32(1.1)); +SELECT round(toFloat64(-2.718281828459), toFloat64(1.1)), ceil(toFloat64(-2.718281828459), toFloat64(1.1)), floor(toFloat64(-2.718281828459), toFloat64(1.1)); + +SELECT round(toFloat64(-2.718281828459), toUInt16(0)), ceil(toFloat64(-2.718281828459), toUInt16(0)), floor(toFloat64(-2.718281828459), toUInt16(0)); +SELECT round(toFloat64(-2.718281828459), toUInt32(0)), ceil(toFloat64(-2.718281828459), toUInt32(0)), floor(toFloat64(-2.718281828459), toUInt32(0)); +SELECT round(toFloat64(-2.718281828459), toUInt64(0)), ceil(toFloat64(-2.718281828459), toUInt64(0)), floor(toFloat64(-2.718281828459), toUInt64(0)); +SELECT round(toFloat64(-2.718281828459), toInt8(0)), ceil(toFloat64(-2.718281828459), toInt8(0)), floor(toFloat64(-2.718281828459), toInt8(0)); +SELECT round(toFloat64(-2.718281828459), toInt16(0)), ceil(toFloat64(-2.718281828459), toInt16(0)), floor(toFloat64(-2.718281828459), toInt16(0)); +SELECT round(toFloat64(-2.718281828459), toInt32(0)), ceil(toFloat64(-2.718281828459), toInt32(0)), floor(toFloat64(-2.718281828459), toInt32(0)); +SELECT round(toFloat64(-2.718281828459), toInt64(0)), ceil(toFloat64(-2.718281828459), toInt64(0)), floor(toFloat64(-2.718281828459), toInt64(0)); +SELECT round(toFloat64(-2.718281828459), toFloat32(0.1)), ceil(toFloat64(-2.718281828459), toFloat32(0.1)), floor(toFloat64(-2.718281828459), toFloat32(0.1)); +SELECT round(toFloat64(-2.718281828459), toFloat64(0.1)), ceil(toFloat64(-2.718281828459), toFloat64(0.1)), floor(toFloat64(-2.718281828459), toFloat64(0.1)); + +SELECT round(toFloat64(-2.718281828459), toInt8(-1)), ceil(toFloat64(-2.718281828459), toInt8(-1)), floor(toFloat64(-2.718281828459), toInt8(-1)); +SELECT round(toFloat64(-2.718281828459), toInt16(-1)), ceil(toFloat64(-2.718281828459), toInt16(-1)), floor(toFloat64(-2.718281828459), toInt16(-1)); +SELECT round(toFloat64(-2.718281828459), toInt32(-1)), ceil(toFloat64(-2.718281828459), toInt32(-1)), floor(toFloat64(-2.718281828459), toInt32(-1)); +SELECT round(toFloat64(-2.718281828459), toInt64(-1)), ceil(toFloat64(-2.718281828459), toInt64(-1)), floor(toFloat64(-2.718281828459), toInt64(-1)); +SELECT round(toFloat64(-2.718281828459), toFloat32(1.1)), ceil(toFloat64(-2.718281828459), toFloat32(-1.1)), floor(toFloat64(-2.718281828459), toFloat32(-1.1)); +SELECT round(toFloat64(-2.718281828459), toFloat64(1.1)), ceil(toFloat64(-2.718281828459), toFloat64(-1.1)), floor(toFloat64(-2.718281828459), toFloat64(-1.1)); + +SELECT round(toFloat64(-2.718281828459), toInt8(-2)), ceil(toFloat64(-2.718281828459), toInt8(-2)), floor(toFloat64(-2.718281828459), toInt8(-2)); +SELECT round(toFloat64(-2.718281828459), toInt16(-2)), ceil(toFloat64(-2.718281828459), toInt16(-2)), floor(toFloat64(-2.718281828459), toInt16(-2)); +SELECT round(toFloat64(-2.718281828459), toInt32(-2)), ceil(toFloat64(-2.718281828459), toInt32(-2)), floor(toFloat64(-2.718281828459), toInt32(-2)); +SELECT round(toFloat64(-2.718281828459), toInt64(-2)), ceil(toFloat64(-2.718281828459), toInt64(-2)), floor(toFloat64(-2.718281828459), toInt64(-2)); +SELECT round(toFloat64(-2.718281828459), toFloat32(-2.1)), ceil(toFloat64(-2.718281828459), toFloat32(-2.1)), floor(toFloat64(-2.718281828459), toFloat32(-2.1)); +SELECT round(toFloat64(-2.718281828459), toFloat64(-2.1)), ceil(toFloat64(-2.718281828459), toFloat64(-2.1)), floor(toFloat64(-2.718281828459), toFloat64(-2.1)); + +/* Misc. */ + +SELECT round(13112221, -1), ceil(13112221, -1), floor(13112221, -1); +SELECT round(13112221, -2), ceil(13112221, -2), floor(13112221, -2); +SELECT round(13112221, -3), ceil(13112221, -3), floor(13112221, -3); +SELECT round(13112221, -4), ceil(13112221, -4), floor(13112221, -4); +SELECT round(13112221, -5), ceil(13112221, -5), floor(13112221, -5); +SELECT round(13112221, -6), ceil(13112221, -6), floor(13112221, -6); +SELECT round(13112221, -7), ceil(13112221, -7), floor(13112221, -7); +SELECT round(13112221, -8), ceil(13112221, -8), floor(13112221, -8); +SELECT round(13112221, -9), ceil(13112221, -9), floor(13112221, -9); +SELECT round(13112221, -10), ceil(13112221, -10), floor(13112221, -10); +SELECT round(13112221, -11), ceil(13112221, -11), floor(13112221, -11); +SELECT round(13112221, -12), ceil(13112221, -12), floor(13112221, -12); +SELECT round(13112221, -13), ceil(13112221, -13), floor(13112221, -13); +SELECT round(13112221, -14), ceil(13112221, -14), floor(13112221, -14); +SELECT round(13112221, -15), ceil(13112221, -15), floor(13112221, -15); +SELECT round(13112221, -16), ceil(13112221, -16), floor(13112221, -16); +SELECT round(13112221, -17), ceil(13112221, -17), floor(13112221, -17); +SELECT round(13112221, -18), ceil(13112221, -18), floor(13112221, -18); +SELECT round(13112221, -19), ceil(13112221, -19), floor(13112221, -19); +SELECT round(13112221, -20), ceil(13112221, -20), floor(13112221, -20); + +SELECT round(2.718281828459045, 1), ceil(2.718281828459045, 1), floor(2.718281828459045, 1); +SELECT round(2.718281828459045, 2), ceil(2.718281828459045, 2), floor(2.718281828459045, 2); +SELECT round(2.718281828459045, 3), ceil(2.718281828459045, 3), floor(2.718281828459045, 3); +SELECT round(2.718281828459045, 4), ceil(2.718281828459045, 4), floor(2.718281828459045, 4); +SELECT round(2.718281828459045, 5), ceil(2.718281828459045, 5), floor(2.718281828459045, 5); +SELECT round(2.718281828459045, 6), ceil(2.718281828459045, 6), floor(2.718281828459045, 6); +SELECT round(2.718281828459045, 7), ceil(2.718281828459045, 7), floor(2.718281828459045, 7); +SELECT round(2.718281828459045, 8), ceil(2.718281828459045, 8), floor(2.718281828459045, 8); +SELECT round(2.718281828459045, 9), ceil(2.718281828459045, 9), floor(2.718281828459045, 9); +SELECT round(2.718281828459045, 10), ceil(2.718281828459045, 10), floor(2.718281828459045, 10); +SELECT round(2.718281828459045, 11), ceil(2.718281828459045, 11), floor(2.718281828459045, 11); +SELECT round(2.718281828459045, 12), ceil(2.718281828459045, 12), floor(2.718281828459045, 12); +SELECT round(2.718281828459045, 13), ceil(2.718281828459045, 13), floor(2.718281828459045, 13); +SELECT round(2.718281828459045, 14), ceil(2.718281828459045, 14), floor(2.718281828459045, 14); +SELECT round(2.718281828459045, 15), ceil(2.718281828459045, 15), floor(2.718281828459045, 15); +SELECT round(2.718281828459045, 16), ceil(2.718281828459045, 16), floor(2.718281828459045, 16); +SELECT round(2.718281828459045, 17), ceil(2.718281828459045, 17), floor(2.718281828459045, 17); +SELECT round(2.718281828459045, 18), ceil(2.718281828459045, 18), floor(2.718281828459045, 18); +SELECT round(2.718281828459045, 19), ceil(2.718281828459045, 19), floor(2.718281828459045, 19); +SELECT round(2.718281828459045, 20), ceil(2.718281828459045, 20), floor(2.718281828459045, 20); + + + From 3bff6644f9eaa05c3abddcb4b07e66a1b7db9c76 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Mon, 25 May 2015 19:32:53 +0300 Subject: [PATCH 023/109] dbms: Server: Updated functional test for rounding functions. [#METR-15210] --- .../00161_rounding_functions.reference | 55 +++++++++++++++++++ .../0_stateless/00161_rounding_functions.sql | 11 +++- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/dbms/tests/queries/0_stateless/00161_rounding_functions.reference b/dbms/tests/queries/0_stateless/00161_rounding_functions.reference index f924a9f5d0d..990df3ca74b 100644 --- a/dbms/tests/queries/0_stateless/00161_rounding_functions.reference +++ b/dbms/tests/queries/0_stateless/00161_rounding_functions.reference @@ -797,3 +797,58 @@ 2.718281828459045 2.718281828459045 2.718281828459045 2.718281828459045 2.718281828459045 2.718281828459045 2.718281828459045 2.718281828459045 2.718281828459045 +3.718 +3.718 +3.218 +3.718 +3.218 +2.918 +3.718 +3.218 +2.918 +2.818 +3.718 +3.218 +2.918 +2.818 +2.777 +3.718 +3.218 +2.918 +2.818 +2.777 +2.757 +3.718 +3.218 +2.918 +2.818 +2.777 +2.757 +2.745 +3.718 +3.218 +2.918 +2.818 +2.777 +2.757 +2.745 +2.738 +3.718 +3.218 +2.918 +2.818 +2.777 +2.757 +2.745 +2.738 +2.734 +3.718 +3.218 +2.918 +2.818 +2.777 +2.757 +2.745 +2.738 +2.734 +2.73 diff --git a/dbms/tests/queries/0_stateless/00161_rounding_functions.sql b/dbms/tests/queries/0_stateless/00161_rounding_functions.sql index 7851f8e02c2..052796b8247 100644 --- a/dbms/tests/queries/0_stateless/00161_rounding_functions.sql +++ b/dbms/tests/queries/0_stateless/00161_rounding_functions.sql @@ -936,5 +936,14 @@ SELECT round(2.718281828459045, 18), ceil(2.718281828459045, 18), floor(2.718281 SELECT round(2.718281828459045, 19), ceil(2.718281828459045, 19), floor(2.718281828459045, 19); SELECT round(2.718281828459045, 20), ceil(2.718281828459045, 20), floor(2.718281828459045, 20); - +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(1) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(2) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(3) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(4) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(5) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(6) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(7) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(8) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(9) AS x); +SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(10) AS x); From c160b7e460ae4251c92e594b4b6e4d43a1e31c34 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 May 2015 03:35:13 +0300 Subject: [PATCH 024/109] dbms: tiny modification [#METR-2944]. --- dbms/include/DB/Storages/MergeTree/PKCondition.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/include/DB/Storages/MergeTree/PKCondition.h b/dbms/include/DB/Storages/MergeTree/PKCondition.h index 838d762cc96..4e33612fc9f 100644 --- a/dbms/include/DB/Storages/MergeTree/PKCondition.h +++ b/dbms/include/DB/Storages/MergeTree/PKCondition.h @@ -128,13 +128,13 @@ private: public: Field left; /// левая граница, если есть Field right; /// правая граница, если есть - bool left_bounded; /// ограничен ли слева - bool right_bounded; /// ограничен ли справа - bool left_included; /// включает левую границу, если есть - bool right_included; /// включает правую границу, если есть + bool left_bounded = false; /// ограничен ли слева + bool right_bounded = false; /// ограничен ли справа + bool left_included = false; /// включает левую границу, если есть + bool right_included = false; /// включает правую границу, если есть /// Всё множество. - Range() : left(), right(), left_bounded(false), right_bounded(false), left_included(false), right_included(false) {} + Range() {} /// Одна точка. Range(const Field & point) : left(point), right(point), left_bounded(true), right_bounded(true), left_included(true), right_included(true) {} From baaf5d04857c6ec3a070246441a5f8a565282a67 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 26 May 2015 03:37:48 +0300 Subject: [PATCH 025/109] dbms: allowed to JOIN with empty table [#METR-16476]. --- dbms/include/DB/Interpreters/Join.h | 7 ++ dbms/src/Interpreters/ExpressionAnalyzer.cpp | 1 + dbms/src/Interpreters/Join.cpp | 65 ++++++++++++++----- dbms/src/Storages/StorageJoin.cpp | 1 + .../0_stateless/00057_join_aliases.reference | 12 ++-- 5 files changed, 63 insertions(+), 23 deletions(-) diff --git a/dbms/include/DB/Interpreters/Join.h b/dbms/include/DB/Interpreters/Join.h index 0d53724e68f..034390e8492 100644 --- a/dbms/include/DB/Interpreters/Join.h +++ b/dbms/include/DB/Interpreters/Join.h @@ -74,6 +74,11 @@ public: bool empty() { return type == Type::EMPTY; } + /** Передать информацию о структуре блока. + * Следует обязательно вызвать до вызовов insertFromBlock. + */ + void setSampleBlock(const Block & block); + /** Добавить в отображение для соединения блок "правой" таблицы. * Возвращает false, если превышено какое-нибудь ограничение, и больше не нужно вставлять. */ @@ -217,6 +222,8 @@ private: bool keys_fit_128_bits; Sizes key_sizes; + Block sample_block; + Logger * log; /// Ограничения на максимальный размер множества diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 593fa39b991..8eb92e2ab09 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -1591,6 +1591,7 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty { auto interpreter = interpretSubquery(ast_join.table, context, subquery_depth, required_joined_columns); subquery_for_set.source = new LazyBlockInputStream([interpreter]() mutable { return interpreter->execute(); }); + join->setSampleBlock(interpreter->getSampleBlock()); } subquery_for_set.join = join; diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 5b37896bfc3..d8abb1453dc 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -323,10 +323,45 @@ void Join::insertFromBlockImpl(Maps & maps, size_t rows, const ConstColumnPlainP } +void Join::setSampleBlock(const Block & block) +{ + Poco::ScopedWriteRWLock lock(rwlock); + + if (!empty()) + return; + + size_t keys_size = key_names_right.size(); + ConstColumnPlainPtrs key_columns(keys_size); + + for (size_t i = 0; i < keys_size; ++i) + key_columns[i] = block.getByName(key_names_right[i]).column; + + /// Выберем, какую структуру данных для множества использовать. + init(chooseMethod(key_columns, keys_fit_128_bits, key_sizes)); + + sample_block = block; + + /// Удаляем из sample_block ключевые столбцы, так как они не нужны. + for (const auto & name : key_names_right) + sample_block.erase(sample_block.getPositionByName(name)); + + for (size_t i = 0, size = sample_block.columns(); i < size; ++i) + { + auto & column = sample_block.unsafeGetByPosition(i); + if (!column.column) + column.column = column.type->createColumn(); + } +} + + bool Join::insertFromBlock(const Block & block) { Poco::ScopedWriteRWLock lock(rwlock); + /// Какую структуру данных для множества использовать? + if (empty()) + throw Exception("Logical error: Join was not initialized", ErrorCodes::LOGICAL_ERROR); + size_t keys_size = key_names_right.size(); ConstColumnPlainPtrs key_columns(keys_size); @@ -336,10 +371,6 @@ bool Join::insertFromBlock(const Block & block) size_t rows = block.rows(); - /// Какую структуру данных для множества использовать? - if (empty()) - init(chooseMethod(key_columns, keys_fit_128_bits, key_sizes)); - blocks.push_back(block); Block * stored_block = &blocks.back(); @@ -347,6 +378,14 @@ bool Join::insertFromBlock(const Block & block) for (const auto & name : key_names_right) stored_block->erase(stored_block->getPositionByName(name)); + /// Редкий случай, когда соединяемые столбцы являеются константами. Чтобы не поддерживать отдельный код, материализуем их. + for (size_t i = 0, size = stored_block->columns(); i < size; ++i) + { + ColumnPtr col = stored_block->getByPosition(i).column; + if (col->isConst()) + stored_block->getByPosition(i).column = dynamic_cast(*col).convertToFullColumn(); + } + if (!getFullness(kind)) { if (strictness == ASTJoin::Any) @@ -473,9 +512,6 @@ struct Adder template void Join::joinBlockImpl(Block & block, const Maps & maps) const { - if (blocks.empty()) - throw Exception("Attempt to JOIN with empty table", ErrorCodes::EMPTY_DATA_PASSED); - size_t keys_size = key_names_left.size(); ConstColumnPlainPtrs key_columns(keys_size); @@ -484,15 +520,14 @@ void Join::joinBlockImpl(Block & block, const Maps & maps) const key_columns[i] = block.getByName(key_names_left[i]).column; /// Добавляем в блок новые столбцы. - const Block & first_mapped_block = blocks.front(); - size_t num_columns_to_add = first_mapped_block.columns(); + size_t num_columns_to_add = sample_block.columns(); ColumnPlainPtrs added_columns(num_columns_to_add); size_t existing_columns = block.columns(); for (size_t i = 0; i < num_columns_to_add; ++i) { - const ColumnWithNameAndType & src_column = first_mapped_block.getByPosition(i); + const ColumnWithNameAndType & src_column = sample_block.getByPosition(i); ColumnWithNameAndType new_column = src_column.cloneEmpty(); block.insert(new_column); added_columns[i] = new_column.column; @@ -630,11 +665,8 @@ void Join::joinTotals(Block & block) const } else { - if (blocks.empty()) - return; - /// Будем присоединять пустые totals - из одной строчки со значениями по-умолчанию. - totals_without_keys = blocks.front().cloneEmpty(); + totals_without_keys = sample_block.cloneEmpty(); for (size_t i = 0; i < totals_without_keys.columns(); ++i) { @@ -739,13 +771,12 @@ private: } /// Добавляем в блок новые столбцы. - const Block & first_mapped_block = parent.blocks.front(); - size_t num_columns_right = first_mapped_block.columns(); + size_t num_columns_right = parent.sample_block.columns(); ColumnPlainPtrs columns_right(num_columns_right); for (size_t i = 0; i < num_columns_right; ++i) { - const ColumnWithNameAndType & src_column = first_mapped_block.getByPosition(i); + const ColumnWithNameAndType & src_column = parent.sample_block.getByPosition(i); ColumnWithNameAndType new_column = src_column.cloneEmpty(); block.insert(new_column); columns_right[i] = new_column.column; diff --git a/dbms/src/Storages/StorageJoin.cpp b/dbms/src/Storages/StorageJoin.cpp index 4460df1e592..8b48205a6c8 100644 --- a/dbms/src/Storages/StorageJoin.cpp +++ b/dbms/src/Storages/StorageJoin.cpp @@ -33,6 +33,7 @@ StorageJoin::StorageJoin( ErrorCodes::NO_SUCH_COLUMN_IN_TABLE}; join = new Join(key_names, key_names, Limits(), kind, strictness); + join->setSampleBlock(getSampleBlock()); restore(); } diff --git a/dbms/tests/queries/0_stateless/00057_join_aliases.reference b/dbms/tests/queries/0_stateless/00057_join_aliases.reference index aedcedebd44..07f91fefd0a 100644 --- a/dbms/tests/queries/0_stateless/00057_join_aliases.reference +++ b/dbms/tests/queries/0_stateless/00057_join_aliases.reference @@ -1,10 +1,10 @@ 0 0 0 Hello -1 0.5 0 Hello +1 0.5 0 2 1 3 Hello -3 1.5 0 Hello +3 1.5 0 4 2 6 Hello -5 2.5 0 Hello +5 2.5 0 6 3 9 Hello -7 3.5 0 Hello -8 4 0 Hello -9 4.5 0 Hello +7 3.5 0 +8 4 0 +9 4.5 0 From 28822deffb981bedaed1290a779a53de1c551ed1 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Tue, 26 May 2015 13:00:17 +0300 Subject: [PATCH 026/109] dbms: Server: Fixed computation overflow issue. [#METR-16435] --- dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index b64771bfeb5..88e9dbd49c5 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -165,7 +165,7 @@ BlockInputStreams MergeTreeDataSelectExecutor::read( UInt64 sampling_column_value_lower_limit; UInt64 sampling_column_value_upper_limit; - UInt64 upper_limit = static_cast(relative_sample_size * sampling_column_max); + UInt64 upper_limit = static_cast(static_cast(relative_sample_size) * sampling_column_max); if (settings.parallel_replicas_count > 1) { From 31b868184d61ee685fc7ceae076c9efa3635f726 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 26 May 2015 14:53:58 +0300 Subject: [PATCH 027/109] dbms: show all dictionaries in system.dictionaries, even the ones failed to load. [#METR-16466] --- .../DB/Interpreters/ExternalDictionaries.h | 31 +++----- .../src/Interpreters/ExternalDictionaries.cpp | 63 ++++++++++++---- .../Storages/StorageSystemDictionaries.cpp | 71 +++++++++++-------- 3 files changed, 102 insertions(+), 63 deletions(-) diff --git a/dbms/include/DB/Interpreters/ExternalDictionaries.h b/dbms/include/DB/Interpreters/ExternalDictionaries.h index 2ad0b9975ec..4bf27d8bbee 100644 --- a/dbms/include/DB/Interpreters/ExternalDictionaries.h +++ b/dbms/include/DB/Interpreters/ExternalDictionaries.h @@ -43,10 +43,14 @@ private: mutable std::mutex dictionaries_mutex; using dictionary_ptr_t = std::shared_ptr>; - using dictionary_origin_pair_t = std::pair; - std::unordered_map dictionaries; - /// exception pointers for notifying user about failures on dictionary creation - std::unordered_map stored_exceptions; + struct dictionary_info final + { + dictionary_ptr_t dict; + std::string origin; + std::exception_ptr exception; + }; + + std::unordered_map dictionaries; std::unordered_map update_times; std::mt19937_64 rnd_engine{getSeed()}; @@ -95,24 +99,7 @@ public: reloading_thread.join(); } - MultiVersion::Version getDictionary(const std::string & name) const - { - const std::lock_guard lock{dictionaries_mutex}; - const auto it = dictionaries.find(name); - if (it == std::end(dictionaries)) - { - const auto exception_it = stored_exceptions.find(name); - if (exception_it != std::end(stored_exceptions)) - std::rethrow_exception(exception_it->second); - else - throw Exception{ - "No such dictionary: " + name, - ErrorCodes::BAD_ARGUMENTS - }; - } - - return it->second.first->get(); - } + MultiVersion::Version getDictionary(const std::string & name) const; }; } diff --git a/dbms/src/Interpreters/ExternalDictionaries.cpp b/dbms/src/Interpreters/ExternalDictionaries.cpp index 3c3f099bcc2..8d62fb6aa3f 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.cpp +++ b/dbms/src/Interpreters/ExternalDictionaries.cpp @@ -48,7 +48,10 @@ void ExternalDictionaries::reloadImpl(const bool throw_on_error) try { - auto current = dictionary.second.first->get(); + if (!dictionary.second.dict) + continue; + + auto current = dictionary.second.dict->get(); const auto & lifetime = current->getLifetime(); /// do not update dictionaries with zero as lifetime @@ -75,16 +78,16 @@ void ExternalDictionaries::reloadImpl(const bool throw_on_error) { /// create new version of dictionary auto new_version = current->clone(); - dictionary.second.first->set(new_version.release()); + dictionary.second.dict->set(new_version.release()); } } /// erase stored exception on success - stored_exceptions.erase(name); + dictionary.second.exception = std::exception_ptr{}; } catch (...) { - stored_exceptions.emplace(name, std::current_exception()); + dictionary.second.exception = std::current_exception(); try { @@ -162,8 +165,8 @@ void ExternalDictionaries::reloadFromFile(const std::string & config_path, const auto it = dictionaries.find(name); if (it != std::end(dictionaries)) - if (it->second.second != config_path) - throw std::runtime_error{"Overriding dictionary from file " + it->second.second}; + if (it->second.origin != config_path) + throw std::runtime_error{"Overriding dictionary from file " + it->second.origin}; auto dict_ptr = DictionaryFactory::instance().create(name, *config, key, context); if (!dict_ptr->isCached()) @@ -184,22 +187,39 @@ void ExternalDictionaries::reloadFromFile(const std::string & config_path, const if (it == std::end(dictionaries)) { const std::lock_guard lock{dictionaries_mutex}; - dictionaries.emplace(name, dictionary_origin_pair_t{ + dictionaries.emplace(name, dictionary_info{ std::make_shared>(dict_ptr.release()), config_path }); } else - it->second.first->set(dict_ptr.release()); + { + if (it->second.dict) + it->second.dict->set(dict_ptr.release()); + else + { + const std::lock_guard lock{dictionaries_mutex}; + it->second.dict = std::make_shared>(dict_ptr.release()); + } - /// erase stored exception on success - stored_exceptions.erase(name); + /// erase stored exception on success + it->second.exception = std::exception_ptr{}; + } } catch (...) { - const auto exception_ptr = std::current_exception(); if (!name.empty()) - stored_exceptions.emplace(name, exception_ptr); + { + const auto exception_ptr = std::current_exception(); + const auto it = dictionaries.find(name); + if (it == std::end(dictionaries)) + { + const std::lock_guard lock{dictionaries_mutex}; + dictionaries.emplace(name, dictionary_info{nullptr, config_path, exception_ptr}); + } + else + it->second.exception = exception_ptr; + } try { @@ -223,11 +243,28 @@ void ExternalDictionaries::reloadFromFile(const std::string & config_path, const /// propagate exception if (throw_on_error) - std::rethrow_exception(exception_ptr); + throw; } } } } } +MultiVersion::Version ExternalDictionaries::getDictionary(const std::string & name) const +{ + const std::lock_guard lock{dictionaries_mutex}; + const auto it = dictionaries.find(name); + + if (it == std::end(dictionaries)) + throw Exception{ + "No such dictionary: " + name, + ErrorCodes::BAD_ARGUMENTS + }; + + if (!it->second.dict && it->second.exception) + std::rethrow_exception(it->second.exception); + + return it->second.dict->get(); +} + } diff --git a/dbms/src/Storages/StorageSystemDictionaries.cpp b/dbms/src/Storages/StorageSystemDictionaries.cpp index 70e215f7bba..bf8f92de366 100644 --- a/dbms/src/Storages/StorageSystemDictionaries.cpp +++ b/dbms/src/Storages/StorageSystemDictionaries.cpp @@ -21,8 +21,8 @@ StorageSystemDictionaries::StorageSystemDictionaries(const std::string & name) : name{name}, columns{ { "name", new DataTypeString }, - { "type", new DataTypeString }, { "origin", new DataTypeString }, + { "type", new DataTypeString }, { "attribute.names", new DataTypeArray{new DataTypeString} }, { "attribute.types", new DataTypeArray{new DataTypeString} }, { "has_hierarchy", new DataTypeUInt8 }, @@ -56,8 +56,8 @@ BlockInputStreams StorageSystemDictionaries::read( processed_stage = QueryProcessingStage::FetchColumns; ColumnWithNameAndType col_name{new ColumnString, new DataTypeString, "name"}; - ColumnWithNameAndType col_type{new ColumnString, new DataTypeString, "type"}; ColumnWithNameAndType col_origin{new ColumnString, new DataTypeString, "origin"}; + ColumnWithNameAndType col_type{new ColumnString, new DataTypeString, "type"}; ColumnWithNameAndType col_attribute_names{ new ColumnArray{new ColumnString}, new DataTypeArray{new DataTypeString}, @@ -83,34 +83,51 @@ BlockInputStreams StorageSystemDictionaries::read( for (const auto & dict_info : external_dictionaries.dictionaries) { - const auto & name = dict_info.first; - const auto dict_ptr = dict_info.second.first->get(); + col_name.column->insert(dict_info.first); + col_origin.column->insert(dict_info.second.origin); - col_name.column->insert(name); - col_type.column->insert(dict_ptr->getTypeName()); - col_origin.column->insert(dict_info.second.second); + if (dict_info.second.dict) + { + const auto dict_ptr = dict_info.second.dict->get(); - const auto & dict_struct = dict_ptr->getStructure(); - col_attribute_names.column->insert(ext::map(dict_struct.attributes, [] (auto & attr) -> decltype(auto) { - return attr.name; - })); - col_attribute_types.column->insert(ext::map(dict_struct.attributes, [] (auto & attr) -> decltype(auto) { - return attr.type->getName(); - })); - col_has_hierarchy.column->insert(UInt64{dict_ptr->hasHierarchy()}); - col_bytes_allocated.column->insert(dict_ptr->getBytesAllocated()); - col_query_count.column->insert(dict_ptr->getQueryCount()); - col_hit_rate.column->insert(dict_ptr->getHitRate()); - col_element_count.column->insert(dict_ptr->getElementCount()); - col_load_factor.column->insert(dict_ptr->getLoadFactor()); - col_creation_time.column->insert(std::chrono::system_clock::to_time_t(dict_ptr->getCreationTime())); + col_type.column->insert(dict_ptr->getTypeName()); - const auto exception_it = external_dictionaries.stored_exceptions.find(name); - if (exception_it != std::end(external_dictionaries.stored_exceptions)) + const auto & dict_struct = dict_ptr->getStructure(); + col_attribute_names.column->insert(ext::map(dict_struct.attributes, [] (auto & attr) -> decltype(auto) { + return attr.name; + })); + col_attribute_types.column->insert(ext::map(dict_struct.attributes, [] (auto & attr) -> decltype(auto) { + return attr.type->getName(); + })); + col_has_hierarchy.column->insert(UInt64{dict_ptr->hasHierarchy()}); + col_bytes_allocated.column->insert(dict_ptr->getBytesAllocated()); + col_query_count.column->insert(dict_ptr->getQueryCount()); + col_hit_rate.column->insert(dict_ptr->getHitRate()); + col_element_count.column->insert(dict_ptr->getElementCount()); + col_load_factor.column->insert(dict_ptr->getLoadFactor()); + col_creation_time.column->insert(std::chrono::system_clock::to_time_t(dict_ptr->getCreationTime())); + col_source.column->insert(dict_ptr->getSource()->toString()); + } + else + { + col_type.column->insertDefault(); + col_attribute_names.column->insertDefault(); + col_attribute_types.column->insertDefault(); + col_has_hierarchy.column->insertDefault(); + col_bytes_allocated.column->insertDefault(); + col_query_count.column->insertDefault(); + col_hit_rate.column->insertDefault(); + col_element_count.column->insertDefault(); + col_load_factor.column->insertDefault(); + col_creation_time.column->insertDefault(); + col_source.column->insertDefault(); + } + + if (dict_info.second.exception) { try { - std::rethrow_exception(exception_it->second); + std::rethrow_exception(dict_info.second.exception); } catch (const Exception & e) { @@ -130,15 +147,13 @@ BlockInputStreams StorageSystemDictionaries::read( } } else - col_last_exception.column->insert(std::string{}); - - col_source.column->insert(dict_ptr->getSource()->toString()); + col_last_exception.column->insertDefault(); } Block block{ col_name, - col_type, col_origin, + col_type, col_attribute_names, col_attribute_types, col_has_hierarchy, From afa79bfcda3ef084ab5e6c9f330dbf160c04c986 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 26 May 2015 15:24:31 +0300 Subject: [PATCH 028/109] dbms: add parsing of expression to DictionaryStructure [#METR-16432] --- .../DB/Dictionaries/DictionaryStructure.h | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/dbms/include/DB/Dictionaries/DictionaryStructure.h b/dbms/include/DB/Dictionaries/DictionaryStructure.h index 5a02bb35928..ca35fe933ba 100644 --- a/dbms/include/DB/Dictionaries/DictionaryStructure.h +++ b/dbms/include/DB/Dictionaries/DictionaryStructure.h @@ -78,7 +78,7 @@ inline std::string toString(const AttributeUnderlyingType type) } /// Min and max lifetimes for a dictionary or it's entry -struct DictionaryLifetime +struct DictionaryLifetime final { std::uint64_t min_sec; std::uint64_t max_sec; @@ -101,18 +101,19 @@ struct DictionaryLifetime * - hierarchical, whether this attribute defines a hierarchy; * - injective, whether the mapping to parent is injective (can be used for optimization of GROUP BY?) */ -struct DictionaryAttribute +struct DictionaryAttribute final { - std::string name; - AttributeUnderlyingType underlying_type; - DataTypePtr type; - Field null_value; - bool hierarchical; - bool injective; + const std::string name; + const AttributeUnderlyingType underlying_type; + const DataTypePtr type; + const std::string expression; + const Field null_value; + const bool hierarchical; + const bool injective; }; /// Name of identifier plus list of attributes -struct DictionaryStructure +struct DictionaryStructure final { std::string id_name; std::vector attributes; @@ -142,6 +143,8 @@ struct DictionaryStructure const auto type = DataTypeFactory::instance().get(type_string); const auto underlying_type = getAttributeUnderlyingType(type_string); + const auto expression = config.getString(prefix + "expression", ""); + const auto null_value_string = config.getString(prefix + "null_value"); Field null_value; try @@ -174,7 +177,7 @@ struct DictionaryStructure has_hierarchy = has_hierarchy || hierarchical; attributes.emplace_back(DictionaryAttribute{ - name, underlying_type, type, null_value, hierarchical, injective + name, underlying_type, type, expression, null_value, hierarchical, injective }); } From ff4fb2784a3d4943aa9216e9e8551c3036959ede Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 26 May 2015 17:40:36 +0300 Subject: [PATCH 029/109] zkutil: rework KeeperException to minimize repetition, set non-zero return code for KeeperException, process ZSESSIONMOVED as unrecoverable --- dbms/include/DB/Core/ErrorCodes.h | 1 + .../include/zkutil/KeeperException.h | 28 +++++++++++-------- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/dbms/include/DB/Core/ErrorCodes.h b/dbms/include/DB/Core/ErrorCodes.h index 6e5ea17341b..c8667e96dc7 100644 --- a/dbms/include/DB/Core/ErrorCodes.h +++ b/dbms/include/DB/Core/ErrorCodes.h @@ -285,6 +285,7 @@ namespace ErrorCodes LEADERSHIP_LOST = 278, ALL_CONNECTION_TRIES_FAILED = 279, + KEEPER_EXCEPTION = 999, POCO_EXCEPTION = 1000, STD_EXCEPTION = 1001, UNKNOWN_EXCEPTION = 1002, diff --git a/libs/libzkutil/include/zkutil/KeeperException.h b/libs/libzkutil/include/zkutil/KeeperException.h index cc3b097eb6a..46ef3948836 100644 --- a/libs/libzkutil/include/zkutil/KeeperException.h +++ b/libs/libzkutil/include/zkutil/KeeperException.h @@ -2,21 +2,26 @@ #include #include #include - +#include namespace zkutil { class KeeperException : public DB::Exception { +private: + /// delegate constructor, used to minimize repetition; last parameter used for overload resolution + KeeperException(const std::string & msg, const int32_t code, int) + : DB::Exception(msg, DB::ErrorCodes::KEEPER_EXCEPTION), code(code) { incrementEventCounter(); } + public: - KeeperException(const std::string & msg) : DB::Exception(msg), code(ZOK) { incrementEventCounter(); } - KeeperException(const std::string & msg, int32_t code_) - : DB::Exception(msg + " (" + zerror(code_) + ")"), code(code_) { incrementEventCounter(); } - KeeperException(int32_t code_) - : DB::Exception(zerror(code_)), code(code_) { incrementEventCounter(); } - KeeperException(int32_t code_, const std::string & path_) - : DB::Exception(std::string(zerror(code_)) + ", path: " + path_), code(code_) { incrementEventCounter(); } + KeeperException(const std::string & msg) : KeeperException(msg, ZOK, 0) {} + KeeperException(const std::string & msg, const int32_t code) + : KeeperException(msg + " (" + zerror(code) + ")", code, 0) {} + KeeperException(const int32_t code) : KeeperException(zerror(code), code, 0) {} + KeeperException(const int32_t code, const std::string & path) + : KeeperException(std::string{zerror(code)} + ", path: " + path, code, 0) {} + KeeperException(const KeeperException & exc) : DB::Exception(exc), code(exc.code) { incrementEventCounter(); } const char * name() const throw() { return "zkutil::KeeperException"; } @@ -26,12 +31,13 @@ public: /// при этих ошибках надо переинициализировать сессию с zookeeper bool isUnrecoverable() const { - return code == ZINVALIDSTATE || code == ZSESSIONEXPIRED; + return code == ZINVALIDSTATE || code == ZSESSIONEXPIRED || code == ZSESSIONMOVED; } - int32_t code; + + const int32_t code; private: - void incrementEventCounter() + static void incrementEventCounter() { ProfileEvents::increment(ProfileEvents::ZooKeeperExceptions); } From 7f38ad390e6c2acaa6ef280da35c9994b90f4dc8 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 26 May 2015 18:09:53 +0300 Subject: [PATCH 030/109] dbms: allow specifying expressions for dictionary attributes [#METR-16432] --- .../Dictionaries/ClickHouseDictionarySource.h | 53 ++++++++++++------- .../DB/Dictionaries/DictionarySourceFactory.h | 4 +- .../DB/Dictionaries/MySQLDictionarySource.h | 53 ++++++++++++------- 3 files changed, 68 insertions(+), 42 deletions(-) diff --git a/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h b/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h index 56671aa774c..e1c47462d95 100644 --- a/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h +++ b/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h @@ -20,10 +20,12 @@ const auto max_connections = 16; class ClickHouseDictionarySource final : public IDictionarySource { public: - ClickHouseDictionarySource(const Poco::Util::AbstractConfiguration & config, + ClickHouseDictionarySource(const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, Context & context) - : host{config.getString(config_prefix + ".host")}, + : dict_struct{dict_struct}, + host{config.getString(config_prefix + ".host")}, port(config.getInt(config_prefix + ".port")), user{config.getString(config_prefix + ".user", "")}, password{config.getString(config_prefix + ".password", "")}, @@ -41,7 +43,8 @@ public: /// copy-constructor is provided in order to support cloneability ClickHouseDictionarySource(const ClickHouseDictionarySource & other) - : host{other.host}, port{other.port}, user{other.user}, password{other.password}, + : dict_struct{other.dict_struct}, + host{other.host}, port{other.port}, user{other.user}, password{other.password}, db{other.db}, table{other.table}, where{other.where}, sample_block{other.sample_block}, context(other.context), @@ -90,14 +93,19 @@ private: WriteBufferFromString out{query}; writeString("SELECT ", out); - auto first = true; - for (const auto idx : ext::range(0, sample_block.columns())) - { - if (!first) - writeString(", ", out); + writeProbablyBackQuotedString(dict_struct.id_name, out); - writeString(sample_block.getByPosition(idx).name, out); - first = false; + for (const auto & attr : dict_struct.attributes) + { + writeString(", ", out); + + if (!attr.expression.empty()) + { + writeString(attr.expression, out); + writeString(" AS ", out); + } + + writeProbablyBackQuotedString(attr.name, out); } writeString(" FROM ", out); @@ -128,17 +136,21 @@ private: WriteBufferFromString out{query}; writeString("SELECT ", out); - auto first = true; - for (const auto idx : ext::range(0, sample_block.columns())) - { - if (!first) - writeString(", ", out); + writeProbablyBackQuotedString(dict_struct.id_name, out); - writeString(sample_block.getByPosition(idx).name, out); - first = false; + for (const auto & attr : dict_struct.attributes) + { + writeString(", ", out); + + if (!attr.expression.empty()) + { + writeString(attr.expression, out); + writeString(" AS ", out); + } + + writeProbablyBackQuotedString(attr.name, out); } - const auto & id_column_name = sample_block.getByPosition(0).name; writeString(" FROM ", out); if (!db.empty()) { @@ -155,10 +167,10 @@ private: writeString(" AND ", out); } - writeProbablyBackQuotedString(id_column_name, out); + writeProbablyBackQuotedString(dict_struct.id_name, out); writeString(" IN (", out); - first = true; + auto first = true; for (const auto id : ids) { if (!first) @@ -174,6 +186,7 @@ private: return query; } + const DictionaryStructure dict_struct; const std::string host; const UInt16 port; const std::string user; diff --git a/dbms/include/DB/Dictionaries/DictionarySourceFactory.h b/dbms/include/DB/Dictionaries/DictionarySourceFactory.h index 865275901f1..fc5992fbe9e 100644 --- a/dbms/include/DB/Dictionaries/DictionarySourceFactory.h +++ b/dbms/include/DB/Dictionaries/DictionarySourceFactory.h @@ -64,11 +64,11 @@ public: } else if ("mysql" == source_type) { - return std::make_unique(config, config_prefix + ".mysql", sample_block); + return std::make_unique(dict_struct, config, config_prefix + ".mysql", sample_block); } else if ("clickhouse" == source_type) { - return std::make_unique(config, config_prefix + ".clickhouse", + return std::make_unique(dict_struct, config, config_prefix + ".clickhouse", sample_block, context); } diff --git a/dbms/include/DB/Dictionaries/MySQLDictionarySource.h b/dbms/include/DB/Dictionaries/MySQLDictionarySource.h index f47ce818102..e5f83232e13 100644 --- a/dbms/include/DB/Dictionaries/MySQLDictionarySource.h +++ b/dbms/include/DB/Dictionaries/MySQLDictionarySource.h @@ -16,9 +16,11 @@ class MySQLDictionarySource final : public IDictionarySource static const auto max_block_size = 8192; public: - MySQLDictionarySource(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, + MySQLDictionarySource(const DictionaryStructure & dict_struct, + const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block) - : db{config.getString(config_prefix + ".db", "")}, + : dict_struct{dict_struct}, + db{config.getString(config_prefix + ".db", "")}, table{config.getString(config_prefix + ".table")}, where{config.getString(config_prefix + ".where", "")}, sample_block{sample_block}, @@ -29,7 +31,8 @@ public: /// copy-constructor is provided in order to support cloneability MySQLDictionarySource(const MySQLDictionarySource & other) - : db{other.db}, + : dict_struct{other.dict_struct}, + db{other.db}, table{other.table}, where{other.where}, sample_block{other.sample_block}, @@ -101,14 +104,19 @@ private: WriteBufferFromString out{query}; writeString("SELECT ", out); - auto first = true; - for (const auto idx : ext::range(0, sample_block.columns())) - { - if (!first) - writeString(", ", out); + writeProbablyBackQuotedString(dict_struct.id_name, out); - writeString(sample_block.getByPosition(idx).name, out); - first = false; + for (const auto & attr : dict_struct.attributes) + { + writeString(", ", out); + + if (!attr.expression.empty()) + { + writeString(attr.expression, out); + writeString(" AS ", out); + } + + writeProbablyBackQuotedString(attr.name, out); } writeString(" FROM ", out); @@ -139,17 +147,21 @@ private: WriteBufferFromString out{query}; writeString("SELECT ", out); - auto first = true; - for (const auto idx : ext::range(0, sample_block.columns())) - { - if (!first) - writeString(", ", out); + writeProbablyBackQuotedString(dict_struct.id_name, out); - writeString(sample_block.getByPosition(idx).name, out); - first = false; + for (const auto & attr : dict_struct.attributes) + { + writeString(", ", out); + + if (!attr.expression.empty()) + { + writeString(attr.expression, out); + writeString(" AS ", out); + } + + writeProbablyBackQuotedString(attr.name, out); } - const auto & id_column_name = sample_block.getByPosition(0).name; writeString(" FROM ", out); if (!db.empty()) { @@ -166,10 +178,10 @@ private: writeString(" AND ", out); } - writeProbablyBackQuotedString(id_column_name, out); + writeProbablyBackQuotedString(dict_struct.id_name, out); writeString(" IN (", out); - first = true; + auto first = true; for (const auto id : ids) { if (!first) @@ -185,6 +197,7 @@ private: return query; } + const DictionaryStructure dict_struct; const std::string db; const std::string table; const std::string where; From 5390f492e107621d5c1443942e0b878f8b0ba453 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 27 May 2015 03:55:54 +0300 Subject: [PATCH 031/109] dbms: JOIN with empty table: additions [#METR-16476]. --- .../DB/Interpreters/ExpressionAnalyzer.h | 1 + dbms/src/Interpreters/ExpressionAnalyzer.cpp | 6 +++++- .../0_stateless/00162_global_join.reference | 2 ++ .../queries/0_stateless/00162_global_join.sql | 1 + .../00163_join_with_empty_table.reference | 20 +++++++++++++++++++ .../00163_join_with_empty_table.sql | 4 ++++ 6 files changed, 33 insertions(+), 1 deletion(-) create mode 100644 dbms/tests/queries/0_stateless/00162_global_join.reference create mode 100644 dbms/tests/queries/0_stateless/00162_global_join.sql create mode 100644 dbms/tests/queries/0_stateless/00163_join_with_empty_table.reference create mode 100644 dbms/tests/queries/0_stateless/00163_join_with_empty_table.sql diff --git a/dbms/include/DB/Interpreters/ExpressionAnalyzer.h b/dbms/include/DB/Interpreters/ExpressionAnalyzer.h index 1acf3a4fb25..42a35830d16 100644 --- a/dbms/include/DB/Interpreters/ExpressionAnalyzer.h +++ b/dbms/include/DB/Interpreters/ExpressionAnalyzer.h @@ -22,6 +22,7 @@ struct SubqueryForSet { /// Источник - получен с помощью InterpreterSelectQuery подзапроса. BlockInputStreamPtr source; + Block source_sample; /// Если задано - создать из результата Set. SetPtr set; diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 8eb92e2ab09..ab25424be59 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -769,6 +769,7 @@ void ExpressionAnalyzer::addExternalStorage(ASTPtr & subquery_or_table_name) external_tables[external_table_name] = external_storage; subqueries_for_sets[external_table_name].source = interpreter->execute(); + subqueries_for_sets[external_table_name].source_sample = interpreter->getSampleBlock(); subqueries_for_sets[external_table_name].table = external_storage; /** NOTE Если было написано IN tmp_table - существующая временная (но не внешняя) таблица, @@ -842,6 +843,7 @@ void ExpressionAnalyzer::makeSet(ASTFunction * node, const Block & sample_block) { auto interpreter = interpretSubquery(arg, context, subquery_depth); subquery_for_set.source = new LazyBlockInputStream([interpreter]() mutable { return interpreter->execute(); }); + subquery_for_set.source_sample = interpreter->getSampleBlock(); /** Зачем используется LazyBlockInputStream? * @@ -1591,10 +1593,12 @@ bool ExpressionAnalyzer::appendJoin(ExpressionActionsChain & chain, bool only_ty { auto interpreter = interpretSubquery(ast_join.table, context, subquery_depth, required_joined_columns); subquery_for_set.source = new LazyBlockInputStream([interpreter]() mutable { return interpreter->execute(); }); - join->setSampleBlock(interpreter->getSampleBlock()); + subquery_for_set.source_sample = interpreter->getSampleBlock(); } + /// TODO Это не нужно выставлять, когда JOIN нужен только на удалённых серверах. subquery_for_set.join = join; + subquery_for_set.join->setSampleBlock(subquery_for_set.source_sample); } addJoinAction(step.actions, false); diff --git a/dbms/tests/queries/0_stateless/00162_global_join.reference b/dbms/tests/queries/0_stateless/00162_global_join.reference new file mode 100644 index 00000000000..1af796ba02c --- /dev/null +++ b/dbms/tests/queries/0_stateless/00162_global_join.reference @@ -0,0 +1,2 @@ +2 6 Hello +2 6 Hello diff --git a/dbms/tests/queries/0_stateless/00162_global_join.sql b/dbms/tests/queries/0_stateless/00162_global_join.sql new file mode 100644 index 00000000000..72e112620ee --- /dev/null +++ b/dbms/tests/queries/0_stateless/00162_global_join.sql @@ -0,0 +1 @@ +SELECT toFloat64(dummy + 2) AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.one) GLOBAL ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 10) USING n LIMIT 10; diff --git a/dbms/tests/queries/0_stateless/00163_join_with_empty_table.reference b/dbms/tests/queries/0_stateless/00163_join_with_empty_table.reference new file mode 100644 index 00000000000..52b5e800b26 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00163_join_with_empty_table.reference @@ -0,0 +1,20 @@ +0 0 0 +1 0.5 0 +2 1 0 +3 1.5 0 +4 2 0 +5 2.5 0 +6 3 0 +7 3.5 0 +8 4 0 +9 4.5 0 +0 0 0 +1 0.5 0 +2 1 0 +3 1.5 0 +4 2 0 +5 2.5 0 +6 3 0 +7 3.5 0 +8 4 0 +9 4.5 0 diff --git a/dbms/tests/queries/0_stateless/00163_join_with_empty_table.sql b/dbms/tests/queries/0_stateless/00163_join_with_empty_table.sql new file mode 100644 index 00000000000..af4e8e79ff6 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00163_join_with_empty_table.sql @@ -0,0 +1,4 @@ +SELECT number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.numbers) ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) USING n LIMIT 10; +SELECT dummy + 2 AS number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.one) ANY INNER JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) USING n LIMIT 10; +SELECT number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.numbers) GLOBAL ANY LEFT JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) USING n LIMIT 10; +SELECT dummy + 2 AS number, number / 2 AS n, j1, j2 FROM remote('127.0.0.{2,3}', system.one) GLOBAL ANY INNER JOIN (SELECT number / 3 AS n, number AS j1, 'Hello' AS j2 FROM system.numbers LIMIT 0) USING n LIMIT 10; From 463ee81a9d85cbf3edce0a5ea40cbe986d5c6da3 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Thu, 28 May 2015 01:59:46 +0300 Subject: [PATCH 032/109] dbms: Server: Performance improvements + fixed rounding algorithm for negative precisions. [#METR-15210] --- dbms/include/DB/Functions/FunctionsRound.h | 203 +++++++++++++++++++-- 1 file changed, 189 insertions(+), 14 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index dc27ad45c32..2a2ebd86307 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -97,6 +97,68 @@ namespace DB } }; + /** Быстрое вычисление остатка от деления для применения к округлению целых чисел. + * Без проверки, потому что делитель всегда положительный. + */ + template + struct FastModulo + { + }; + + template + struct FastModulo::value>::type> + { + private: + template + struct Extend + { + }; + + template + struct Extend::value + || std::is_same::value>::type> + { + using Type = Int64; + }; + + template + struct Extend::value + || std::is_same::value>::type> + { + using Type = UInt64; + }; + + template + struct Extend::value + && (sizeof(InputType) >= 4)>::type> + { + using Type = InputType; + }; + + using U = typename Extend::Type; + + public: + using Divisor = std::pair >; + + static inline Divisor prepare(size_t b) + { + return std::make_pair(b, libdivide::divider(b)); + } + + static inline T compute(T a, const Divisor & divisor) + { + if (divisor.first == 1) + return 0; + + U val = static_cast(a); + U rem = val - (val / divisor.second) * static_cast(divisor.first); + return static_cast(rem); + } + }; + /** Этот параметр контролирует поведение функций округления. */ enum ScaleMode @@ -119,7 +181,14 @@ namespace DB typename std::enable_if::value && ((scale_mode == PositiveScale) || (scale_mode == ZeroScale))>::type> { - static inline T compute(const T in, size_t scale) + using Divisor = int; + + static inline Divisor prepare(size_t scale) + { + return 0; + } + + static inline T compute(T in, const Divisor & scale) { return in; } @@ -129,14 +198,26 @@ namespace DB struct IntegerRoundingComputation::value>::type> { - static inline T compute(T in, size_t scale) + using Op = FastModulo; + using Divisor = typename Op::Divisor; + + static inline Divisor prepare(size_t scale) { - T rem = in % scale; + return Op::prepare(scale); + } + + static inline T compute(T in, const Divisor & scale) + { + T factor = (in < 0) ? -1 : 1; + in *= factor; + T rem = Op::compute(in, scale); in -= rem; - if (static_cast(2 * rem) < scale) - return in; + T res; + if ((2 * rem) < static_cast(scale.first)) + res = in; else - return in + scale; + res = in + scale.first; + return factor * res; } }; @@ -144,10 +225,21 @@ namespace DB struct IntegerRoundingComputation::value>::type> { - static inline T compute(const T in, size_t scale) + using Op = FastModulo; + using Divisor = typename Op::Divisor; + + static inline Divisor prepare(size_t scale) { - T rem = in % scale; - return in - rem + scale; + return Op::prepare(scale); + } + + static inline T compute(T in, const Divisor & scale) + { + T factor = (in < 0) ? -1 : 1; + in *= factor; + T rem = Op::compute(in, scale); + T res = in - rem + scale.first; + return factor * res; } }; @@ -155,10 +247,21 @@ namespace DB struct IntegerRoundingComputation::value>::type> { - static inline T compute(const T in, size_t scale) + using Op = FastModulo; + using Divisor = typename Op::Divisor; + + static inline Divisor prepare(size_t scale) { - T rem = in % scale; - return in - rem; + return Op::prepare(scale); + } + + static inline T compute(T in, const Divisor & scale) + { + T factor = (in < 0) ? -1 : 1; + in *= factor; + T rem = Op::compute(in, scale); + T res = in - rem; + return factor * res; } }; @@ -221,11 +324,28 @@ namespace DB static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) { __m128 val = _mm_loadu_ps(in); + + /// Превратить отрицательные значения в положительные. + __m128 factor = _mm_cmpge_ps(val, getZero()); + factor = _mm_min_ps(factor, getTwo()); + factor = _mm_sub_ps(factor, getOne()); + val = _mm_mul_ps(val, factor); + + /// Алгоритм округления. val = _mm_div_ps(val, scale); __m128 res = _mm_cmpge_ps(val, getOneTenth()); val = _mm_round_ps(val, rounding_mode); val = _mm_mul_ps(val, scale); val = _mm_and_ps(val, res); + + /// Предотвратить появление отрицательных нолей определённых в стандарте IEEE-754. + __m128 check = _mm_cmpeq_ps(val, getZero()); + check = _mm_min_ps(check, getOne()); + factor = _mm_add_ps(factor, check); + + /// Вернуть настоящие знаки всех значений. + val = _mm_mul_ps(val, factor); + _mm_storeu_ps(out, val); } @@ -235,6 +355,24 @@ namespace DB static const __m128 one_tenth = _mm_set1_ps(0.1); return one_tenth; } + + static inline const __m128 & getZero() + { + static const __m128 zero = _mm_set1_ps(0.0); + return zero; + } + + static inline const __m128 & getOne() + { + static const __m128 one = _mm_set1_ps(1.0); + return one; + } + + static inline const __m128 & getTwo() + { + static const __m128 two = _mm_set1_ps(2.0); + return two; + } }; template @@ -286,11 +424,28 @@ namespace DB static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) { __m128d val = _mm_loadu_pd(in); + + /// Превратить отрицательные значения в положительные. + __m128d factor = _mm_cmpge_pd(val, getZero()); + factor = _mm_min_pd(factor, getTwo()); + factor = _mm_sub_pd(factor, getOne()); + val = _mm_mul_pd(val, factor); + + /// Алгоритм округления. val = _mm_div_pd(val, scale); __m128d res = _mm_cmpge_pd(val, getOneTenth()); val = _mm_round_pd(val, rounding_mode); val = _mm_mul_pd(val, scale); val = _mm_and_pd(val, res); + + /// Предотвратить появление отрицательных нолей определённых в стандарте IEEE-754. + __m128d check = _mm_cmpeq_pd(val, getZero()); + check = _mm_min_pd(check, getOne()); + factor = _mm_add_pd(factor, check); + + /// Вернуть настоящие знаки всех значений. + val = _mm_mul_pd(val, factor); + _mm_storeu_pd(out, val); } @@ -300,6 +455,24 @@ namespace DB static const __m128d one_tenth = _mm_set1_pd(0.1); return one_tenth; } + + static inline const __m128d & getZero() + { + static const __m128d zero = _mm_set1_pd(0.0); + return zero; + } + + static inline const __m128d & getOne() + { + static const __m128d one = _mm_set1_pd(1.0); + return one; + } + + static inline const __m128d & getTwo() + { + static const __m128d two = _mm_set1_pd(2.0); + return two; + } }; template @@ -337,14 +510,16 @@ namespace DB public: static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { + auto divisor = Op::prepare(scale); size_t size = in.size(); for (size_t i = 0; i < size; ++i) - out[i] = Op::compute(in[i], scale); + out[i] = Op::compute(in[i], divisor); } static inline T apply(T val, size_t scale) { - return Op::compute(val, scale); + auto divisor = Op::prepare(scale); + return Op::compute(val, divisor); } }; From ef54e74f4556e4558d2702e5d1cdbf2195dc6e7c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 May 2015 04:41:40 +0300 Subject: [PATCH 033/109] dbms: supported 'NOT NOT x' chain in parser [#METR-16543]. --- dbms/src/Parsers/ExpressionListParsers.cpp | 31 +++++++++++++++++++ .../0_stateless/00164_not_chain.reference | 5 +++ .../queries/0_stateless/00164_not_chain.sql | 5 +++ 3 files changed, 41 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00164_not_chain.reference create mode 100644 dbms/tests/queries/0_stateless/00164_not_chain.sql diff --git a/dbms/src/Parsers/ExpressionListParsers.cpp b/dbms/src/Parsers/ExpressionListParsers.cpp index 404f25cd14b..0fc7ab7fead 100644 --- a/dbms/src/Parsers/ExpressionListParsers.cpp +++ b/dbms/src/Parsers/ExpressionListParsers.cpp @@ -375,6 +375,37 @@ bool ParserPrefixUnaryOperatorExpression::parseImpl(Pos & pos, Pos end, ASTPtr & ws.ignore(pos, end); + /// Позволяем парсить цепочки вида NOT NOT x. Это хак. + /** Так сделано, потому что среди унарных операторов есть только минус и NOT. + * Но для минуса цепочку из унарных операторов не требуется поддерживать. + */ + if (it[0] && 0 == strncmp(it[0], "NOT", 3)) + { + /// Было ли чётное количество NOT. + bool even = false; + + const char ** jt; + while (true) + { + for (jt = operators; *jt; jt += 2) + { + ParserString op(jt[0], true, true); + if (op.ignore(pos, end, max_parsed_pos, expected)) + break; + } + + if (!*jt) + break; + + even = !even; + + ws.ignore(pos, end); + } + + if (even) + it = jt; /// Зануляем результат парсинга первого NOT. Получается, как будто цепочки NOT нет вообще. + } + ASTPtr elem; if (!elem_parser->parse(pos, end, elem, max_parsed_pos, expected)) return false; diff --git a/dbms/tests/queries/0_stateless/00164_not_chain.reference b/dbms/tests/queries/0_stateless/00164_not_chain.reference new file mode 100644 index 00000000000..ac8d6d16066 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00164_not_chain.reference @@ -0,0 +1,5 @@ +0 +1 +0 +1 +0 diff --git a/dbms/tests/queries/0_stateless/00164_not_chain.sql b/dbms/tests/queries/0_stateless/00164_not_chain.sql new file mode 100644 index 00000000000..39fe8724e33 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00164_not_chain.sql @@ -0,0 +1,5 @@ +SELECT NOT 1; +SELECT NOT NOT 1; +SELECT NOT NOT NOT 1; +SELECT NOT NOT NOT NOT 1 = 1; +SELECT NOT NOT not NoT NOT 1 = 1; From 0984363b9c0c74049792485139c9cab34121f797 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 May 2015 06:49:28 +0300 Subject: [PATCH 034/109] dbms: removed passing DataTypeFactory everywhere [#METR-16545]. --- dbms/include/DB/Client/Connection.h | 7 +------ dbms/include/DB/Client/ConnectionPool.h | 7 ++----- dbms/include/DB/Common/ExternalTable.h | 6 ++++-- dbms/include/DB/Core/NamesAndTypes.h | 5 ++--- dbms/include/DB/DataStreams/FormatFactory.h | 6 ++---- .../DB/DataStreams/MergeSortingBlockInputStream.h | 9 ++++----- .../DB/DataStreams/NativeBlockInputStream.h | 6 ++---- .../DB/Dictionaries/ClickHouseDictionarySource.h | 4 ++-- .../include/DB/Dictionaries/FileDictionarySource.h | 2 +- dbms/include/DB/Interpreters/Cluster.h | 8 +++----- dbms/include/DB/Interpreters/Context.h | 1 - .../DB/Interpreters/InterpreterAlterQuery.h | 2 +- dbms/include/DB/Storages/ColumnsDescription.h | 2 +- .../DB/Storages/Distributed/DirectoryMonitor.h | 2 +- dbms/include/DB/Storages/MergeTree/MergeTreeData.h | 2 +- .../DB/Storages/MergeTree/MergeTreePartChecker.h | 3 +-- dbms/include/DB/Storages/StorageSet.h | 2 +- .../DB/TableFunctions/TableFunctionRemote.h | 7 +++++-- dbms/src/Client/Benchmark.cpp | 3 +-- dbms/src/Client/Client.cpp | 4 ++-- dbms/src/Client/Connection.cpp | 14 +++++++------- dbms/src/Core/NamesAndTypes.cpp | 10 +++++++--- dbms/src/DataStreams/FormatFactory.cpp | 4 ++-- .../DataStreams/MergeSortingBlockInputStream.cpp | 2 +- dbms/src/DataStreams/NativeBlockInputStream.cpp | 3 +++ dbms/src/DataStreams/tests/native_streams.cpp | 3 --- dbms/src/DataStreams/tests/sorting_stream.cpp | 3 +-- dbms/src/Interpreters/Cluster.cpp | 14 +++++++------- dbms/src/Interpreters/Context.cpp | 5 +---- dbms/src/Interpreters/InterpreterAlterQuery.cpp | 7 +++++-- dbms/src/Interpreters/InterpreterCreateQuery.cpp | 6 +++++- dbms/src/Interpreters/InterpreterInsertQuery.cpp | 3 +-- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 2 +- .../src/Interpreters/tests/expression_analyzer.cpp | 3 ++- dbms/src/Server/TCPHandler.cpp | 1 - dbms/src/Storages/ColumnsDescription.cpp | 4 +++- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../Storages/MergeTree/MergeTreePartChecker.cpp | 12 +++++------- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 9 ++++----- dbms/src/Storages/StorageSet.cpp | 8 +++----- dbms/src/Storages/tests/part_checker.cpp | 2 +- 41 files changed, 97 insertions(+), 108 deletions(-) diff --git a/dbms/include/DB/Client/Connection.h b/dbms/include/DB/Client/Connection.h index 9f81b5a7feb..2d40c2e6141 100644 --- a/dbms/include/DB/Client/Connection.h +++ b/dbms/include/DB/Client/Connection.h @@ -12,8 +12,6 @@ #include #include -#include - #include #include #include @@ -50,7 +48,6 @@ class Connection : private boost::noncopyable public: Connection(const String & host_, UInt16 port_, const String & default_database_, const String & user_, const String & password_, - const DataTypeFactory & data_type_factory_, const String & client_name_ = "client", Protocol::Compression::Enum compression_ = Protocol::Compression::Enable, Poco::Timespan connect_timeout_ = Poco::Timespan(DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, 0), @@ -61,7 +58,7 @@ public: host(host_), port(port_), default_database(default_database_), user(user_), password(password_), client_name(client_name_), - compression(compression_), data_type_factory(data_type_factory_), + compression(compression_), connect_timeout(connect_timeout_), receive_timeout(receive_timeout_), send_timeout(send_timeout_), ping_timeout(ping_timeout_), log_wrapper(host, port) @@ -172,8 +169,6 @@ private: /// каким алгоритмом сжимать данные при INSERT и данные внешних таблиц CompressionMethod network_compression_method = CompressionMethod::LZ4; - const DataTypeFactory & data_type_factory; - /** Если не nullptr, то используется, чтобы ограничить сетевой трафик. * Учитывается только трафик при передаче блоков. Другие пакеты не учитываются. */ diff --git a/dbms/include/DB/Client/ConnectionPool.h b/dbms/include/DB/Client/ConnectionPool.h index 11a6eb14afa..aa779815bf7 100644 --- a/dbms/include/DB/Client/ConnectionPool.h +++ b/dbms/include/DB/Client/ConnectionPool.h @@ -56,7 +56,6 @@ public: ConnectionPool(unsigned max_connections_, const String & host_, UInt16 port_, const String & default_database_, const String & user_, const String & password_, - const DataTypeFactory & data_type_factory_, const String & client_name_ = "client", Protocol::Compression::Enum compression_ = Protocol::Compression::Enable, Poco::Timespan connect_timeout_ = Poco::Timespan(DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, 0), @@ -65,7 +64,7 @@ public: : Base(max_connections_, &Logger::get("ConnectionPool (" + Poco::Net::SocketAddress(host_, port_).toString() + ")")), host(host_), port(port_), default_database(default_database_), user(user_), password(password_), - client_name(client_name_), compression(compression_), data_type_factory(data_type_factory_), + client_name(client_name_), compression(compression_), connect_timeout(connect_timeout_), receive_timeout(receive_timeout_), send_timeout(send_timeout_) { } @@ -91,7 +90,7 @@ protected: { return new Connection( host, port, default_database, user, password, - data_type_factory, client_name, compression, + client_name, compression, connect_timeout, receive_timeout, send_timeout); } @@ -105,8 +104,6 @@ private: String client_name; Protocol::Compression::Enum compression; /// Сжимать ли данные при взаимодействии с сервером. - const DataTypeFactory & data_type_factory; - Poco::Timespan connect_timeout; Poco::Timespan receive_timeout; Poco::Timespan send_timeout; diff --git a/dbms/include/DB/Common/ExternalTable.h b/dbms/include/DB/Common/ExternalTable.h index 4c6ff8c010a..bd984397211 100644 --- a/dbms/include/DB/Common/ExternalTable.h +++ b/dbms/include/DB/Common/ExternalTable.h @@ -42,11 +42,13 @@ public: /// Инициализировать sample_block по структуре таблицы сохраненной в structure virtual void initSampleBlock(const Context & context) { + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + for (size_t i = 0; i < structure.size(); ++i) { ColumnWithNameAndType column; column.name = structure[i].first; - column.type = context.getDataTypeFactory().get(structure[i].second); + column.type = data_type_factory.get(structure[i].second); column.column = column.type->createColumn(); sample_block.insert(column); } @@ -58,7 +60,7 @@ public: initReadBuffer(); initSampleBlock(context); ExternalTableData res = std::make_pair(new AsynchronousBlockInputStream(context.getFormatFactory().getInput( - format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE, context.getDataTypeFactory())), name); + format, *read_buffer, sample_block, DEFAULT_BLOCK_SIZE)), name); return res; } diff --git a/dbms/include/DB/Core/NamesAndTypes.h b/dbms/include/DB/Core/NamesAndTypes.h index ad3c60defa7..3669164b1d3 100644 --- a/dbms/include/DB/Core/NamesAndTypes.h +++ b/dbms/include/DB/Core/NamesAndTypes.h @@ -9,7 +9,6 @@ #include #include -#include #include #include "Names.h" @@ -45,11 +44,11 @@ class NamesAndTypesList : public std::list public: using std::list::list; - void readText(ReadBuffer & buf, const DataTypeFactory & data_type_factory); + void readText(ReadBuffer & buf); void writeText(WriteBuffer & buf) const; String toString() const; - static NamesAndTypesList parse(const String & s, const DataTypeFactory & data_type_factory); + static NamesAndTypesList parse(const String & s); /// Все элементы rhs должны быть различны. bool isSubsetOf(const NamesAndTypesList & rhs) const; diff --git a/dbms/include/DB/DataStreams/FormatFactory.h b/dbms/include/DB/DataStreams/FormatFactory.h index 1b51b8af153..d8e224e1cb0 100644 --- a/dbms/include/DB/DataStreams/FormatFactory.h +++ b/dbms/include/DB/DataStreams/FormatFactory.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include @@ -16,8 +14,8 @@ class FormatFactory { public: BlockInputStreamPtr getInput(const String & name, ReadBuffer & buf, - Block & sample, size_t max_block_size, const DataTypeFactory & data_type_factory) const; - + Block & sample, size_t max_block_size) const; + BlockOutputStreamPtr getOutput(const String & name, WriteBuffer & buf, Block & sample) const; }; diff --git a/dbms/include/DB/DataStreams/MergeSortingBlockInputStream.h b/dbms/include/DB/DataStreams/MergeSortingBlockInputStream.h index a87d81a01f3..7a51097af10 100644 --- a/dbms/include/DB/DataStreams/MergeSortingBlockInputStream.h +++ b/dbms/include/DB/DataStreams/MergeSortingBlockInputStream.h @@ -66,9 +66,9 @@ public: /// limit - если не 0, то можно выдать только первые limit строк в сортированном порядке. MergeSortingBlockInputStream(BlockInputStreamPtr input_, SortDescription & description_, size_t max_merged_block_size_, size_t limit_, - size_t max_bytes_before_external_sort_, const std::string & tmp_path_, const DataTypeFactory & data_type_factory_) + size_t max_bytes_before_external_sort_, const std::string & tmp_path_) : description(description_), max_merged_block_size(max_merged_block_size_), limit(limit_), - max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_path(tmp_path_), data_type_factory(data_type_factory_) + max_bytes_before_external_sort(max_bytes_before_external_sort_), tmp_path(tmp_path_) { children.push_back(input_); } @@ -97,7 +97,6 @@ private: size_t max_bytes_before_external_sort; const std::string tmp_path; - const DataTypeFactory & data_type_factory; Logger * log = &Logger::get("MergeSortingBlockInputStream"); @@ -115,8 +114,8 @@ private: CompressedReadBuffer compressed_in; BlockInputStreamPtr block_in; - TemporaryFileStream(const std::string & path, const DataTypeFactory & data_type_factory) - : file_in(path), compressed_in(file_in), block_in(new NativeBlockInputStream(compressed_in, data_type_factory)) {} + TemporaryFileStream(const std::string & path) + : file_in(path), compressed_in(file_in), block_in(new NativeBlockInputStream(compressed_in)) {} }; std::vector> temporary_inputs; diff --git a/dbms/include/DB/DataStreams/NativeBlockInputStream.h b/dbms/include/DB/DataStreams/NativeBlockInputStream.h index ee387680627..cb277a57939 100644 --- a/dbms/include/DB/DataStreams/NativeBlockInputStream.h +++ b/dbms/include/DB/DataStreams/NativeBlockInputStream.h @@ -1,6 +1,5 @@ #pragma once -#include #include @@ -16,8 +15,8 @@ public: /** В случае указания ненулевой server_revision, может ожидаться и считываться дополнительная информация о блоке, * в зависимости от поддерживаемой для указанной ревизии. */ - NativeBlockInputStream(ReadBuffer & istr_, const DataTypeFactory & data_type_factory_, UInt64 server_revision_ = 0) - : istr(istr_), data_type_factory(data_type_factory_), server_revision(server_revision_) {} + NativeBlockInputStream(ReadBuffer & istr_, UInt64 server_revision_ = 0) + : istr(istr_), server_revision(server_revision_) {} String getName() const override { return "NativeBlockInputStream"; } @@ -35,7 +34,6 @@ protected: private: ReadBuffer & istr; - const DataTypeFactory & data_type_factory; UInt64 server_revision; }; diff --git a/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h b/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h index e1c47462d95..e57248de428 100644 --- a/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h +++ b/dbms/include/DB/Dictionaries/ClickHouseDictionarySource.h @@ -35,7 +35,7 @@ public: sample_block{sample_block}, context(context), is_local{isLocalAddress({ host, port })}, pool{is_local ? nullptr : std::make_unique( - max_connections, host, port, db, user, password, context.getDataTypeFactory(), + max_connections, host, port, db, user, password, "ClickHouseDictionarySource") }, load_all_query{composeLoadAllQuery()} @@ -50,7 +50,7 @@ public: sample_block{other.sample_block}, context(other.context), is_local{other.is_local}, pool{is_local ? nullptr : std::make_unique( - max_connections, host, port, db, user, password, context.getDataTypeFactory(), + max_connections, host, port, db, user, password, "ClickHouseDictionarySource")}, load_all_query{other.load_all_query} {} diff --git a/dbms/include/DB/Dictionaries/FileDictionarySource.h b/dbms/include/DB/Dictionaries/FileDictionarySource.h index cadb3fb085c..41e55f64de5 100644 --- a/dbms/include/DB/Dictionaries/FileDictionarySource.h +++ b/dbms/include/DB/Dictionaries/FileDictionarySource.h @@ -34,7 +34,7 @@ public: { auto in_ptr = std::make_unique(filename); auto stream = context.getFormatFactory().getInput( - format, *in_ptr, sample_block, max_block_size, context.getDataTypeFactory()); + format, *in_ptr, sample_block, max_block_size); last_modification = getLastModification(); return new OwningBufferBlockInputStream{stream, std::move(in_ptr)}; diff --git a/dbms/include/DB/Interpreters/Cluster.h b/dbms/include/DB/Interpreters/Cluster.h index 1bb84dabb9a..0bd30781269 100644 --- a/dbms/include/DB/Interpreters/Cluster.h +++ b/dbms/include/DB/Interpreters/Cluster.h @@ -2,7 +2,6 @@ #include #include -#include #include #include #include @@ -16,10 +15,10 @@ namespace DB class Cluster : private boost::noncopyable { public: - Cluster(const Settings & settings, const DataTypeFactory & data_type_factory, const String & cluster_name); + Cluster(const Settings & settings, const String & cluster_name); /// Построить кластер по именам шардов и реплик. Локальные обрабатываются так же как удаленные. - Cluster(const Settings & settings, const DataTypeFactory & data_type_factory, std::vector> names, + Cluster(const Settings & settings, std::vector> names, const String & username, const String & password); /// количество узлов clickhouse сервера, расположенных локально @@ -98,8 +97,7 @@ struct Clusters typedef std::map Impl; Impl impl; - Clusters(const Settings & settings, const DataTypeFactory & data_type_factory, - const String & config_name = "remote_servers"); + Clusters(const Settings & settings, const String & config_name = "remote_servers"); }; } diff --git a/dbms/include/DB/Interpreters/Context.h b/dbms/include/DB/Interpreters/Context.h index 2fb66570878..f646c627ecb 100644 --- a/dbms/include/DB/Interpreters/Context.h +++ b/dbms/include/DB/Interpreters/Context.h @@ -160,7 +160,6 @@ public: const TableFunctionFactory & getTableFunctionFactory() const; const AggregateFunctionFactory & getAggregateFunctionFactory() const; - const DataTypeFactory & getDataTypeFactory() const; const FormatFactory & getFormatFactory() const; const Dictionaries & getDictionaries() const; const ExternalDictionaries & getExternalDictionaries() const; diff --git a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h index d6a4ec264f9..4ead0b62955 100644 --- a/dbms/include/DB/Interpreters/InterpreterAlterQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterAlterQuery.h @@ -77,7 +77,7 @@ private: Context context; - static void parseAlter(const ASTAlterQuery::ParameterContainer & params, const DataTypeFactory & data_type_factory, + static void parseAlter(const ASTAlterQuery::ParameterContainer & params, AlterCommands & out_alter_commands, PartitionCommands & out_partition_commands); }; diff --git a/dbms/include/DB/Storages/ColumnsDescription.h b/dbms/include/DB/Storages/ColumnsDescription.h index 5816a8bebe6..aa7b08fa394 100644 --- a/dbms/include/DB/Storages/ColumnsDescription.h +++ b/dbms/include/DB/Storages/ColumnsDescription.h @@ -21,7 +21,7 @@ struct ColumnsDescription String toString() const; - static ColumnsDescription parse(const String & str, const DataTypeFactory & data_type_factory); + static ColumnsDescription parse(const String & str); }; diff --git a/dbms/include/DB/Storages/Distributed/DirectoryMonitor.h b/dbms/include/DB/Storages/Distributed/DirectoryMonitor.h index a7b6f7cf74c..52cbfebe936 100644 --- a/dbms/include/DB/Storages/Distributed/DirectoryMonitor.h +++ b/dbms/include/DB/Storages/Distributed/DirectoryMonitor.h @@ -126,7 +126,7 @@ private: const std::string & user, const std::string & password) { return new ConnectionPool{ 1, host, port, "", - user, password, storage.context.getDataTypeFactory(), + user, password, storage.getName() + '_' + name}; }; diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h index 8605f10a42e..8497279cf7d 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h @@ -477,7 +477,7 @@ public: } ReadBufferFromFile file(path, std::min(static_cast(DBMS_DEFAULT_BUFFER_SIZE), Poco::File(path).getSize())); - columns.readText(file, storage.context.getDataTypeFactory()); + columns.readText(file); } void checkNotBroken(bool require_part_metadata) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreePartChecker.h b/dbms/include/DB/Storages/MergeTree/MergeTreePartChecker.h index bf653f2a7bb..8561f62627d 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreePartChecker.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreePartChecker.h @@ -28,8 +28,7 @@ public: * - Проверяет правильность засечек. * Бросает исключение, если кусок испорчен или если проверить не получилось (TODO: можно попробовать разделить эти случаи). */ - static void checkDataPart(String path, const Settings & settings, const DataTypeFactory & data_type_factory, - MergeTreeData::DataPart::Checksums * out_checksums = nullptr); + static void checkDataPart(String path, const Settings & settings, MergeTreeData::DataPart::Checksums * out_checksums = nullptr); }; } diff --git a/dbms/include/DB/Storages/StorageSet.h b/dbms/include/DB/Storages/StorageSet.h index 7b1ade807e5..fa3f87a6a6a 100644 --- a/dbms/include/DB/Storages/StorageSet.h +++ b/dbms/include/DB/Storages/StorageSet.h @@ -45,7 +45,7 @@ protected: void restore(); private: - void restoreFromFile(const String & file_path, const DataTypeFactory & data_type_factory); + void restoreFromFile(const String & file_path); /// Вставить блок в состояние. virtual void insertBlock(const Block & block) = 0; diff --git a/dbms/include/DB/TableFunctions/TableFunctionRemote.h b/dbms/include/DB/TableFunctions/TableFunctionRemote.h index 0939186cf60..f407d70027f 100644 --- a/dbms/include/DB/TableFunctions/TableFunctionRemote.h +++ b/dbms/include/DB/TableFunctions/TableFunctionRemote.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -117,7 +118,7 @@ public: if (names.empty()) throw Exception("Shard list is empty after parsing first argument", ErrorCodes::BAD_ARGUMENTS); - SharedPtr cluster = new Cluster(context.getSettings(), context.getDataTypeFactory(), names, username, password); + SharedPtr cluster = new Cluster(context.getSettings(), names, username, password); return StorageDistributed::create(getName(), chooseColumns(*cluster, remote_database, remote_table, context), remote_database, remote_table, cluster, context); @@ -140,6 +141,8 @@ private: }; input->readPrefix(); + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + while (true) { Block current = input->read(); @@ -153,7 +156,7 @@ private: String column_name = (*name)[i].get(); String data_type_name = (*type)[i].get(); - res.emplace_back(column_name, context.getDataTypeFactory().get(data_type_name)); + res.emplace_back(column_name, data_type_factory.get(data_type_name)); } } diff --git a/dbms/src/Client/Benchmark.cpp b/dbms/src/Client/Benchmark.cpp index 5cfb0c4086a..4c88eebae41 100644 --- a/dbms/src/Client/Benchmark.cpp +++ b/dbms/src/Client/Benchmark.cpp @@ -52,7 +52,7 @@ public: const String & host_, UInt16 port_, const String & default_database_, const String & user_, const String & password_, const Settings & settings_) : concurrency(concurrency_), delay(delay_), queue(concurrency), - connections(concurrency, host_, port_, default_database_, user_, password_, data_type_factory), + connections(concurrency, host_, port_, default_database_, user_, password_), settings(settings_), pool(concurrency) { std::cerr << std::fixed << std::setprecision(3); @@ -73,7 +73,6 @@ private: typedef ConcurrentBoundedQueue Queue; Queue queue; - DataTypeFactory data_type_factory; ConnectionPool connections; Settings settings; diff --git a/dbms/src/Client/Client.cpp b/dbms/src/Client/Client.cpp index f3bcedf88df..6039609e75d 100644 --- a/dbms/src/Client/Client.cpp +++ b/dbms/src/Client/Client.cpp @@ -336,7 +336,7 @@ private: << (!user.empty() ? " as user " + user : "") << "." << std::endl; - connection = new Connection(host, port, default_database, user, password, context.getDataTypeFactory(), "client", compression, + connection = new Connection(host, port, default_database, user, password, "client", compression, Poco::Timespan(config().getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0), Poco::Timespan(config().getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0), Poco::Timespan(config().getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0)); @@ -698,7 +698,7 @@ private: current_format = insert->format; BlockInputStreamPtr block_input = context.getFormatFactory().getInput( - current_format, buf, sample, insert_format_max_block_size, context.getDataTypeFactory()); + current_format, buf, sample, insert_format_max_block_size); BlockInputStreamPtr async_block_input = new AsynchronousBlockInputStream(block_input); diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 9bf7de70ebf..9e22eda4225 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -166,30 +166,30 @@ void Connection::forceConnected() struct PingTimeoutSetter { - PingTimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & ping_timeout_) + PingTimeoutSetter(Poco::Net::StreamSocket & socket_, const Poco::Timespan & ping_timeout_) : socket(socket_), ping_timeout(ping_timeout_) { old_send_timeout = socket.getSendTimeout(); old_receive_timeout = socket.getReceiveTimeout(); - + if (old_send_timeout > ping_timeout) socket.setSendTimeout(ping_timeout); if (old_receive_timeout > ping_timeout) socket.setReceiveTimeout(ping_timeout); } - + ~PingTimeoutSetter() { socket.setSendTimeout(old_send_timeout); socket.setReceiveTimeout(old_receive_timeout); } - + Poco::Net::StreamSocket & socket; Poco::Timespan ping_timeout; Poco::Timespan old_send_timeout; Poco::Timespan old_receive_timeout; }; - + bool Connection::ping() { // LOG_TRACE(log_wrapper.get(), "Ping (" << getServerAddress() << ")"); @@ -237,7 +237,7 @@ bool Connection::ping() void Connection::sendQuery(const String & query, const String & query_id_, UInt64 stage, const Settings * settings, bool with_pending_data) { network_compression_method = settings ? settings->network_compression_method.value : CompressionMethod::LZ4; - + forceConnected(); query_id = query_id_; @@ -494,7 +494,7 @@ void Connection::initBlockInput() else maybe_compressed_in = in; - block_in = new NativeBlockInputStream(*maybe_compressed_in, data_type_factory, server_revision); + block_in = new NativeBlockInputStream(*maybe_compressed_in, server_revision); } } diff --git a/dbms/src/Core/NamesAndTypes.cpp b/dbms/src/Core/NamesAndTypes.cpp index d70e6a9932a..57dfe42dd0b 100644 --- a/dbms/src/Core/NamesAndTypes.cpp +++ b/dbms/src/Core/NamesAndTypes.cpp @@ -1,10 +1,14 @@ #include +#include + namespace DB { -void NamesAndTypesList::readText(ReadBuffer & buf, const DataTypeFactory & data_type_factory) +void NamesAndTypesList::readText(ReadBuffer & buf) { + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + DB::assertString("columns format version: 1\n", buf); size_t count; DB::readText(count, buf); @@ -45,11 +49,11 @@ String NamesAndTypesList::toString() const return s; } -NamesAndTypesList NamesAndTypesList::parse(const String & s, const DataTypeFactory & data_type_factory) +NamesAndTypesList NamesAndTypesList::parse(const String & s) { ReadBufferFromString in(s); NamesAndTypesList res; - res.readText(in, data_type_factory); + res.readText(in); assertEOF(in); return res; } diff --git a/dbms/src/DataStreams/FormatFactory.cpp b/dbms/src/DataStreams/FormatFactory.cpp index 8f265dc2737..6db0e393731 100644 --- a/dbms/src/DataStreams/FormatFactory.cpp +++ b/dbms/src/DataStreams/FormatFactory.cpp @@ -25,10 +25,10 @@ namespace DB { BlockInputStreamPtr FormatFactory::getInput(const String & name, ReadBuffer & buf, - Block & sample, size_t max_block_size, const DataTypeFactory & data_type_factory) const + Block & sample, size_t max_block_size) const { if (name == "Native") - return new NativeBlockInputStream(buf, data_type_factory); + return new NativeBlockInputStream(buf); else if (name == "TabSeparated") return new BlockInputStreamFromRowInputStream(new TabSeparatedRowInputStream(buf, sample), sample, max_block_size); else if (name == "RowBinary") diff --git a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp index 1ed7eb40c03..b56832894d8 100644 --- a/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp +++ b/dbms/src/DataStreams/MergeSortingBlockInputStream.cpp @@ -65,7 +65,7 @@ Block MergeSortingBlockInputStream::readImpl() /// Сформируем сортированные потоки для слияния. for (const auto & file : temporary_files) { - temporary_inputs.emplace_back(new TemporaryFileStream(file->path(), data_type_factory)); + temporary_inputs.emplace_back(new TemporaryFileStream(file->path())); inputs_to_merge.emplace_back(temporary_inputs.back()->block_in); } diff --git a/dbms/src/DataStreams/NativeBlockInputStream.cpp b/dbms/src/DataStreams/NativeBlockInputStream.cpp index 078e7a2e1e7..b9fa55aa449 100644 --- a/dbms/src/DataStreams/NativeBlockInputStream.cpp +++ b/dbms/src/DataStreams/NativeBlockInputStream.cpp @@ -5,6 +5,7 @@ #include #include +#include #include @@ -44,6 +45,8 @@ Block NativeBlockInputStream::readImpl() { Block res; + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + if (istr.eof()) return res; diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/src/DataStreams/tests/native_streams.cpp index 5a2709d4eb4..d23796c54c9 100644 --- a/dbms/src/DataStreams/tests/native_streams.cpp +++ b/dbms/src/DataStreams/tests/native_streams.cpp @@ -15,7 +15,6 @@ #include #include #include -#include #include #include @@ -117,8 +116,6 @@ int main(int argc, char ** argv) /// читаем данные из native файла и одновременно пишем в таблицу if (argc == 2 && 0 == strcmp(argv[1], "write")) { - DataTypeFactory factory; - ReadBufferFromFileDescriptor in1(STDIN_FILENO); CompressedReadBuffer in2(in1); NativeBlockInputStream in3(in2, factory, Revision::get()); diff --git a/dbms/src/DataStreams/tests/sorting_stream.cpp b/dbms/src/DataStreams/tests/sorting_stream.cpp index d19931f98f3..b503ae6ec7d 100644 --- a/dbms/src/DataStreams/tests/sorting_stream.cpp +++ b/dbms/src/DataStreams/tests/sorting_stream.cpp @@ -148,11 +148,10 @@ int main(int argc, char ** argv) sort_columns.push_back(SortColumnDescription(3, 1)); QueryProcessingStage::Enum stage; - DataTypeFactory data_type_factory; Poco::SharedPtr in = table->read(column_names, 0, Context{}, Settings(), stage, argc == 2 ? atoi(argv[1]) : 1048576)[0]; in = new PartialSortingBlockInputStream(in, sort_columns); - in = new MergeSortingBlockInputStream(in, sort_columns, DEFAULT_BLOCK_SIZE, 0, 0, "", data_type_factory); + in = new MergeSortingBlockInputStream(in, sort_columns, DEFAULT_BLOCK_SIZE, 0, 0, ""); //in = new LimitBlockInputStream(in, 10); WriteBufferFromOStream ob(std::cout); diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 51e0a130947..9b71d0a7cdf 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -47,7 +47,7 @@ namespace } -Clusters::Clusters(const Settings & settings, const DataTypeFactory & data_type_factory, const String & config_name) +Clusters::Clusters(const Settings & settings, const String & config_name) { Poco::Util::AbstractConfiguration & config = Poco::Util::Application::instance().config(); Poco::Util::AbstractConfiguration::Keys config_keys; @@ -56,11 +56,11 @@ Clusters::Clusters(const Settings & settings, const DataTypeFactory & data_type_ for (Poco::Util::AbstractConfiguration::Keys::const_iterator it = config_keys.begin(); it != config_keys.end(); ++it) impl.emplace(std::piecewise_construct, std::forward_as_tuple(*it), - std::forward_as_tuple(settings, data_type_factory, config_name + "." + *it)); + std::forward_as_tuple(settings, config_name + "." + *it)); } -Cluster::Cluster(const Settings & settings, const DataTypeFactory & data_type_factory, const String & cluster_name) +Cluster::Cluster(const Settings & settings, const String & cluster_name) { Poco::Util::AbstractConfiguration & config = Poco::Util::Application::instance().config(); Poco::Util::AbstractConfiguration::Keys config_keys; @@ -179,7 +179,7 @@ Cluster::Cluster(const Settings & settings, const DataTypeFactory & data_type_fa replicas.emplace_back(new ConnectionPool( settings.distributed_connections_pool_size, replica.host_port.host().toString(), replica.host_port.port(), "", replica.user, replica.password, - data_type_factory, "server", Protocol::Compression::Enable, + "server", Protocol::Compression::Enable, saturate(settings.connect_timeout_with_failover_ms, settings.limits.max_execution_time), saturate(settings.receive_timeout, settings.limits.max_execution_time), saturate(settings.send_timeout, settings.limits.max_execution_time))); @@ -205,7 +205,7 @@ Cluster::Cluster(const Settings & settings, const DataTypeFactory & data_type_fa pools.emplace_back(new ConnectionPool( settings.distributed_connections_pool_size, address.host_port.host().toString(), address.host_port.port(), "", address.user, address.password, - data_type_factory, "server", Protocol::Compression::Enable, + "server", Protocol::Compression::Enable, saturate(settings.connect_timeout, settings.limits.max_execution_time), saturate(settings.receive_timeout, settings.limits.max_execution_time), saturate(settings.send_timeout, settings.limits.max_execution_time))); @@ -217,7 +217,7 @@ Cluster::Cluster(const Settings & settings, const DataTypeFactory & data_type_fa } -Cluster::Cluster(const Settings & settings, const DataTypeFactory & data_type_factory, std::vector> names, +Cluster::Cluster(const Settings & settings, std::vector> names, const String & username, const String & password) { for (const auto & shard : names) @@ -238,7 +238,7 @@ Cluster::Cluster(const Settings & settings, const DataTypeFactory & data_type_fa replicas.emplace_back(new ConnectionPool( settings.distributed_connections_pool_size, replica.host_port.host().toString(), replica.host_port.port(), "", replica.user, replica.password, - data_type_factory, "server", Protocol::Compression::Enable, + "server", Protocol::Compression::Enable, saturate(settings.connect_timeout_with_failover_ms, settings.limits.max_execution_time), saturate(settings.receive_timeout, settings.limits.max_execution_time), saturate(settings.send_timeout, settings.limits.max_execution_time))); diff --git a/dbms/src/Interpreters/Context.cpp b/dbms/src/Interpreters/Context.cpp index 47a346ba4f7..c67c36e2cd0 100644 --- a/dbms/src/Interpreters/Context.cpp +++ b/dbms/src/Interpreters/Context.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -72,7 +71,6 @@ struct ContextShared Databases databases; /// Список БД и таблиц в них. TableFunctionFactory table_function_factory; /// Табличные функции. AggregateFunctionFactory aggregate_function_factory; /// Агрегатные функции. - DataTypeFactory data_type_factory; /// Типы данных. FormatFactory format_factory; /// Форматы. mutable SharedPtr dictionaries; /// Словари Метрики. Инициализируются лениво. mutable SharedPtr external_dictionaries; @@ -155,7 +153,6 @@ Context::~Context() = default; const TableFunctionFactory & Context::getTableFunctionFactory() const { return shared->table_function_factory; } const AggregateFunctionFactory & Context::getAggregateFunctionFactory() const { return shared->aggregate_function_factory; } -const DataTypeFactory & Context::getDataTypeFactory() const { return shared->data_type_factory; } const FormatFactory & Context::getFormatFactory() const { return shared->format_factory; } InterserverIOHandler & Context::getInterserverIOHandler() { return shared->interserver_io_handler; } Poco::Mutex & Context::getMutex() const { return shared->mutex; } @@ -800,7 +797,7 @@ void Context::initClusters() { Poco::ScopedLock lock(shared->mutex); if (!shared->clusters) - shared->clusters = new Clusters(settings, shared->data_type_factory); + shared->clusters = new Clusters(settings); } Cluster & Context::getCluster(const std::string & cluster_name) diff --git a/dbms/src/Interpreters/InterpreterAlterQuery.cpp b/dbms/src/Interpreters/InterpreterAlterQuery.cpp index adfaf7573ef..c5e140abe0b 100644 --- a/dbms/src/Interpreters/InterpreterAlterQuery.cpp +++ b/dbms/src/Interpreters/InterpreterAlterQuery.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include @@ -35,7 +36,7 @@ void InterpreterAlterQuery::execute() AlterCommands alter_commands; PartitionCommands partition_commands; - parseAlter(alter.parameters, context.getDataTypeFactory(), alter_commands, partition_commands); + parseAlter(alter.parameters, alter_commands, partition_commands); for (const PartitionCommand & command : partition_commands) { @@ -71,9 +72,11 @@ void InterpreterAlterQuery::execute() } void InterpreterAlterQuery::parseAlter( - const ASTAlterQuery::ParameterContainer & params_container, const DataTypeFactory & data_type_factory, + const ASTAlterQuery::ParameterContainer & params_container, AlterCommands & out_alter_commands, PartitionCommands & out_partition_commands) { + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + for (const auto & params : params_container) { if (params.type == ASTAlterQuery::ADD_COLUMN) diff --git a/dbms/src/Interpreters/InterpreterCreateQuery.cpp b/dbms/src/Interpreters/InterpreterCreateQuery.cpp index ae74e1f1807..0ba822511b2 100644 --- a/dbms/src/Interpreters/InterpreterCreateQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCreateQuery.cpp @@ -25,9 +25,11 @@ #include #include #include + #include #include #include +#include namespace DB @@ -272,6 +274,8 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( ASTPtr default_expr_list{new ASTExpressionList}; default_expr_list->children.reserve(column_list_ast.children.size()); + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); + for (auto & ast : column_list_ast.children) { auto & col_decl = typeid_cast(*ast); @@ -280,7 +284,7 @@ InterpreterCreateQuery::ColumnsAndDefaults InterpreterCreateQuery::parseColumns( { const auto & type_range = col_decl.type->range; columns.emplace_back(col_decl.name, - context.getDataTypeFactory().get({ type_range.first, type_range.second })); + data_type_factory.get({ type_range.first, type_range.second })); } else /// we're creating dummy DataTypeUInt8 in order to prevent the NullPointerException in ExpressionActions diff --git a/dbms/src/Interpreters/InterpreterInsertQuery.cpp b/dbms/src/Interpreters/InterpreterInsertQuery.cpp index 12a216e0764..04ab185b338 100644 --- a/dbms/src/Interpreters/InterpreterInsertQuery.cpp +++ b/dbms/src/Interpreters/InterpreterInsertQuery.cpp @@ -114,8 +114,7 @@ void InterpreterInsertQuery::execute(ReadBuffer * remaining_data_istr) BlockInputStreamPtr in{ context.getFormatFactory().getInput( - format, istr, sample, context.getSettings().max_insert_block_size, - context.getDataTypeFactory())}; + format, istr, sample, context.getSettings().max_insert_block_size)}; copyData(*in, *out); } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 00430dd88fa..63de3f21ba0 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -887,7 +887,7 @@ void InterpreterSelectQuery::executeOrder(BlockInputStreams & streams) /// Сливаем сортированные блоки. stream = new MergeSortingBlockInputStream( stream, order_descr, settings.max_block_size, limit, - settings.limits.max_bytes_before_external_sort, context.getTemporaryPath(), context.getDataTypeFactory()); + settings.limits.max_bytes_before_external_sort, context.getTemporaryPath()); } diff --git a/dbms/src/Interpreters/tests/expression_analyzer.cpp b/dbms/src/Interpreters/tests/expression_analyzer.cpp index e63b073e942..48a11137fd4 100644 --- a/dbms/src/Interpreters/tests/expression_analyzer.cpp +++ b/dbms/src/Interpreters/tests/expression_analyzer.cpp @@ -3,6 +3,7 @@ #include #include #include +#include int main(int argc, char ** argv) @@ -25,7 +26,7 @@ int main(int argc, char ** argv) { NameAndTypePair col; col.name = argv[i]; - col.type = context.getDataTypeFactory().get(argv[i + 1]); + col.type = DataTypeFactory::instance().get(argv[i + 1]); columns.push_back(col); } diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp index 1961e0c0b9b..1b784cd3505 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/src/Server/TCPHandler.cpp @@ -606,7 +606,6 @@ void TCPHandler::initBlockInput() state.block_in = new NativeBlockInputStream( *state.maybe_compressed_in, - query_context.getDataTypeFactory(), client_revision); } } diff --git a/dbms/src/Storages/ColumnsDescription.cpp b/dbms/src/Storages/ColumnsDescription.cpp index 977cc74d312..49a50d3c2b2 100644 --- a/dbms/src/Storages/ColumnsDescription.cpp +++ b/dbms/src/Storages/ColumnsDescription.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace DB @@ -50,7 +51,7 @@ String ColumnsDescription::toString() const template <> -ColumnsDescription ColumnsDescription::parse(const String & str, const DataTypeFactory & data_type_factory) +ColumnsDescription ColumnsDescription::parse(const String & str) { ReadBufferFromString buf{str}; @@ -60,6 +61,7 @@ ColumnsDescription ColumnsDescription::parse(const String & str, con assertString(" columns:\n", buf); ParserTernaryOperatorExpression expr_parser; + const DataTypeFactory & data_type_factory = DataTypeFactory::instance(); ColumnsDescription result{}; for (size_t i = 0; i < count; ++i) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 7086c6ee88a..abac777d7a0 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1037,7 +1037,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const St MergeTreePartChecker::Settings settings; settings.setIndexGranularity(index_granularity); settings.setRequireColumnFiles(true); - MergeTreePartChecker::checkDataPart(full_path + relative_path, settings, context.getDataTypeFactory(), &part->checksums); + MergeTreePartChecker::checkDataPart(full_path + relative_path, settings, &part->checksums); { WriteBufferFromFile out(full_path + relative_path + "/checksums.txt.tmp", 4096); diff --git a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp index 5138b9bf457..83300b4a6ac 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp @@ -249,8 +249,7 @@ static size_t checkColumn(const String & path, const String & name, DataTypePtr } } -void MergeTreePartChecker::checkDataPart(String path, const Settings & settings, const DataTypeFactory & data_type_factory, - MergeTreeData::DataPart::Checksums * out_checksums) +void MergeTreePartChecker::checkDataPart(String path, const Settings & settings, MergeTreeData::DataPart::Checksums * out_checksums) { if (!path.empty() && path.back() != '/') path += "/"; @@ -262,7 +261,7 @@ void MergeTreePartChecker::checkDataPart(String path, const Settings & settings, { ReadBufferFromFile buf(path + "columns.txt"); - columns.readText(buf, data_type_factory); + columns.readText(buf); assertEOF(buf); } @@ -275,12 +274,11 @@ void MergeTreePartChecker::checkDataPart(String path, const Settings & settings, /// Реальные чексуммы по содержимому данных. Их несоответствие checksums_txt будет говорить о битых данных. MergeTreeData::DataPart::Checksums checksums_data; - size_t primary_idx_size; { ReadBufferFromFile file_buf(path + "primary.idx"); HashingReadBuffer hashing_buf(file_buf); - primary_idx_size = hashing_buf.tryIgnore(std::numeric_limits::max()); + size_t primary_idx_size = hashing_buf.tryIgnore(std::numeric_limits::max()); checksums_data.files["primary.idx"] = MergeTreeData::DataPart::Checksums::Checksum(primary_idx_size, hashing_buf.getHash()); } @@ -345,9 +343,9 @@ void MergeTreePartChecker::checkDataPart(String path, const Settings & settings, if (rows == Stream::UNKNOWN) throw Exception("No columns", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); - if (primary_idx_size % ((rows - 1) / settings.index_granularity + 1)) +/* if (primary_idx_size % ((rows - 1) / settings.index_granularity + 1)) throw Exception("primary.idx size (" + toString(primary_idx_size) + ") not divisible by number of marks (" - + toString(rows) + "/" + toString(settings.index_granularity) + " rounded up)", ErrorCodes::CORRUPTED_DATA); + + toString(rows) + "/" + toString(settings.index_granularity) + " rounded up)", ErrorCodes::CORRUPTED_DATA);*/ if (settings.require_checksums || !checksums_txt.files.empty()) checksums_txt.checkEqual(checksums_data, true); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index 294b61651fc..c03eefa60cb 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -268,8 +268,7 @@ void StorageReplicatedMergeTree::checkTableStructure(bool skip_sanity_checks, bo assertEOF(buf); zkutil::Stat stat; - auto columns_desc = ColumnsDescription::parse( - zookeeper->get(zookeeper_path + "/columns", &stat), context.getDataTypeFactory()); + auto columns_desc = ColumnsDescription::parse(zookeeper->get(zookeeper_path + "/columns", &stat)); auto & columns = columns_desc.columns; auto & materialized_columns = columns_desc.materialized; @@ -1459,7 +1458,7 @@ void StorageReplicatedMergeTree::alterThread() zkutil::Stat stat; const String columns_str = zookeeper->get(zookeeper_path + "/columns", &stat, alter_thread_event); - auto columns_desc = ColumnsDescription::parse(columns_str, context.getDataTypeFactory()); + auto columns_desc = ColumnsDescription::parse(columns_str); auto & columns = columns_desc.columns; auto & materialized_columns = columns_desc.materialized; @@ -1809,7 +1808,7 @@ void StorageReplicatedMergeTree::partCheckThread() zk_checksums.checkEqual(part->checksums, true); auto zk_columns = NamesAndTypesList::parse( - zookeeper->get(replica_path + "/parts/" + part_name + "/columns"), context.getDataTypeFactory()); + zookeeper->get(replica_path + "/parts/" + part_name + "/columns")); if (part->columns != zk_columns) throw Exception("Columns of local part " + part_name + " are different from ZooKeeper"); @@ -1818,7 +1817,7 @@ void StorageReplicatedMergeTree::partCheckThread() settings.setRequireChecksums(true); settings.setRequireColumnFiles(true); MergeTreePartChecker::checkDataPart( - data.getFullPath() + part_name, settings, context.getDataTypeFactory()); + data.getFullPath() + part_name, settings); LOG_INFO(log, "Part " << part_name << " looks good."); } diff --git a/dbms/src/Storages/StorageSet.cpp b/dbms/src/Storages/StorageSet.cpp index f92e13e0fdb..8923c40dcd4 100644 --- a/dbms/src/Storages/StorageSet.cpp +++ b/dbms/src/Storages/StorageSet.cpp @@ -87,8 +87,6 @@ void StorageSetOrJoinBase::restore() constexpr auto file_suffix = ".bin"; constexpr auto file_suffix_size = strlen(file_suffix); - DataTypeFactory data_type_factory; - Poco::DirectoryIterator dir_end; for (Poco::DirectoryIterator dir_it(path); dir_end != dir_it; ++dir_it) { @@ -104,17 +102,17 @@ void StorageSetOrJoinBase::restore() if (file_num > increment) increment = file_num; - restoreFromFile(dir_it->path(), data_type_factory); + restoreFromFile(dir_it->path()); } } } -void StorageSetOrJoinBase::restoreFromFile(const String & file_path, const DataTypeFactory & data_type_factory) +void StorageSetOrJoinBase::restoreFromFile(const String & file_path) { ReadBufferFromFile backup_buf(file_path); CompressedReadBuffer compressed_backup_buf(backup_buf); - NativeBlockInputStream backup_stream(compressed_backup_buf, data_type_factory); + NativeBlockInputStream backup_stream(compressed_backup_buf); backup_stream.readPrefix(); while (Block block = backup_stream.read()) diff --git a/dbms/src/Storages/tests/part_checker.cpp b/dbms/src/Storages/tests/part_checker.cpp index 2d58812030c..57a66f3dbd5 100644 --- a/dbms/src/Storages/tests/part_checker.cpp +++ b/dbms/src/Storages/tests/part_checker.cpp @@ -23,7 +23,7 @@ int main(int argc, char ** argv) settings.setRequireColumnFiles(argv[2][0] == '1'); settings.setVerbose(true); - DB::MergeTreePartChecker::checkDataPart(argv[1], settings, DB::DataTypeFactory()); + DB::MergeTreePartChecker::checkDataPart(argv[1], settings); } catch (...) { From c03cec07deb55ecd023ed7256b22e082366b8d97 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 May 2015 07:07:09 +0300 Subject: [PATCH 035/109] dbms: addition to prev. revision [#METR-16545]. --- dbms/src/DataStreams/tests/native_streams.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/src/DataStreams/tests/native_streams.cpp b/dbms/src/DataStreams/tests/native_streams.cpp index d23796c54c9..b1e978493a4 100644 --- a/dbms/src/DataStreams/tests/native_streams.cpp +++ b/dbms/src/DataStreams/tests/native_streams.cpp @@ -118,7 +118,7 @@ int main(int argc, char ** argv) { ReadBufferFromFileDescriptor in1(STDIN_FILENO); CompressedReadBuffer in2(in1); - NativeBlockInputStream in3(in2, factory, Revision::get()); + NativeBlockInputStream in3(in2, Revision::get()); SharedPtr out = table->write(0); copyData(in3, *out); } From 986f7b55bb124f533e7d8ec97d52b3d92da6c512 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 May 2015 07:09:09 +0300 Subject: [PATCH 036/109] dbms: addition to prev. revision [#METR-16545]. --- dbms/include/DB/Common/ExternalTable.h | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/include/DB/Common/ExternalTable.h b/dbms/include/DB/Common/ExternalTable.h index bd984397211..43f3e838a8b 100644 --- a/dbms/include/DB/Common/ExternalTable.h +++ b/dbms/include/DB/Common/ExternalTable.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include From 40f0a9c4d9a5923e1e3909fd3d717676151cd729 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 28 May 2015 07:32:38 +0300 Subject: [PATCH 037/109] dbms: fixed error with false-broken parts [#METR-16545]. --- .../DB/Storages/MergeTree/MergeTreeData.h | 2 +- .../Storages/MergeTree/MergeTreePartChecker.h | 6 ++- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MergeTreePartChecker.cpp | 41 ++++++++++++++++--- .../Storages/StorageReplicatedMergeTree.cpp | 2 +- dbms/src/Storages/tests/part_checker.cpp | 10 +++-- 6 files changed, 50 insertions(+), 13 deletions(-) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h index 8497279cf7d..73cf5907a7f 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h @@ -853,13 +853,13 @@ public: const MergeTreeSettings settings; const ASTPtr primary_expr_ast; + Block primary_key_sample; private: bool require_part_metadata; ExpressionActionsPtr primary_expr; SortDescription sort_descr; - Block primary_key_sample; String full_path; diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreePartChecker.h b/dbms/include/DB/Storages/MergeTree/MergeTreePartChecker.h index 8561f62627d..ba7dfcca8fb 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreePartChecker.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreePartChecker.h @@ -28,7 +28,11 @@ public: * - Проверяет правильность засечек. * Бросает исключение, если кусок испорчен или если проверить не получилось (TODO: можно попробовать разделить эти случаи). */ - static void checkDataPart(String path, const Settings & settings, MergeTreeData::DataPart::Checksums * out_checksums = nullptr); + static void checkDataPart( + String path, + const Settings & settings, + const Block & primary_key_sample, /// Проверять первичный ключ. Если не надо - передайте пустой Block. + MergeTreeData::DataPart::Checksums * out_checksums = nullptr); }; } diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index abac777d7a0..57946e82f64 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -1037,7 +1037,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeData::loadPartAndFixMetadata(const St MergeTreePartChecker::Settings settings; settings.setIndexGranularity(index_granularity); settings.setRequireColumnFiles(true); - MergeTreePartChecker::checkDataPart(full_path + relative_path, settings, &part->checksums); + MergeTreePartChecker::checkDataPart(full_path + relative_path, settings, primary_key_sample, &part->checksums); { WriteBufferFromFile out(full_path + relative_path + "/checksums.txt.tmp", 4096); diff --git a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp index 83300b4a6ac..c8e8a84f3fe 100644 --- a/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreePartChecker.cpp @@ -249,7 +249,11 @@ static size_t checkColumn(const String & path, const String & name, DataTypePtr } } -void MergeTreePartChecker::checkDataPart(String path, const Settings & settings, MergeTreeData::DataPart::Checksums * out_checksums) +void MergeTreePartChecker::checkDataPart( + String path, + const Settings & settings, + const Block & primary_key_sample, + MergeTreeData::DataPart::Checksums * out_checksums) { if (!path.empty() && path.back() != '/') path += "/"; @@ -275,10 +279,29 @@ void MergeTreePartChecker::checkDataPart(String path, const Settings & settings, /// Реальные чексуммы по содержимому данных. Их несоответствие checksums_txt будет говорить о битых данных. MergeTreeData::DataPart::Checksums checksums_data; + size_t marks_in_primary_key = 0; { ReadBufferFromFile file_buf(path + "primary.idx"); HashingReadBuffer hashing_buf(file_buf); - size_t primary_idx_size = hashing_buf.tryIgnore(std::numeric_limits::max()); + + if (primary_key_sample) + { + Field tmp_field; + size_t key_size = primary_key_sample.columns(); + while (!hashing_buf.eof()) + { + ++marks_in_primary_key; + for (size_t j = 0; j < key_size; ++j) + primary_key_sample.unsafeGetByPosition(j).type->deserializeBinary(tmp_field, hashing_buf); + } + } + else + { + hashing_buf.tryIgnore(std::numeric_limits::max()); + } + + size_t primary_idx_size = hashing_buf.count(); + checksums_data.files["primary.idx"] = MergeTreeData::DataPart::Checksums::Checksum(primary_idx_size, hashing_buf.getHash()); } @@ -343,9 +366,17 @@ void MergeTreePartChecker::checkDataPart(String path, const Settings & settings, if (rows == Stream::UNKNOWN) throw Exception("No columns", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); -/* if (primary_idx_size % ((rows - 1) / settings.index_granularity + 1)) - throw Exception("primary.idx size (" + toString(primary_idx_size) + ") not divisible by number of marks (" - + toString(rows) + "/" + toString(settings.index_granularity) + " rounded up)", ErrorCodes::CORRUPTED_DATA);*/ + if (primary_key_sample) + { + const size_t expected_marks = (rows - 1) / settings.index_granularity + 1; + if (expected_marks != marks_in_primary_key) + throw Exception("Size of primary key doesn't match expected number of marks." + " Number of rows in columns: " + toString(rows) + + ", index_granularity: " + toString(settings.index_granularity) + + ", expected number of marks: " + toString(expected_marks) + + ", size of primary key: " + toString(marks_in_primary_key), + ErrorCodes::CORRUPTED_DATA); + } if (settings.require_checksums || !checksums_txt.files.empty()) checksums_txt.checkEqual(checksums_data, true); diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index c03eefa60cb..ea7f0072b71 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -1817,7 +1817,7 @@ void StorageReplicatedMergeTree::partCheckThread() settings.setRequireChecksums(true); settings.setRequireColumnFiles(true); MergeTreePartChecker::checkDataPart( - data.getFullPath() + part_name, settings); + data.getFullPath() + part_name, settings, data.primary_key_sample); LOG_INFO(log, "Part " << part_name << " looks good."); } diff --git a/dbms/src/Storages/tests/part_checker.cpp b/dbms/src/Storages/tests/part_checker.cpp index 57a66f3dbd5..1d323ab84a5 100644 --- a/dbms/src/Storages/tests/part_checker.cpp +++ b/dbms/src/Storages/tests/part_checker.cpp @@ -4,6 +4,8 @@ int main(int argc, char ** argv) { + using namespace DB; + Poco::AutoPtr channel = new Poco::ConsoleChannel(std::cerr); Logger::root().setChannel(channel); Logger::root().setLevel("trace"); @@ -16,18 +18,18 @@ int main(int argc, char ** argv) try { - DB::MergeTreePartChecker::Settings settings; + MergeTreePartChecker::Settings settings; if (argc == 4) - settings.setIndexGranularity(DB::parse(argv[3])); + settings.setIndexGranularity(parse(argv[3])); settings.setRequireChecksums(argv[2][0] == '1'); settings.setRequireColumnFiles(argv[2][0] == '1'); settings.setVerbose(true); - DB::MergeTreePartChecker::checkDataPart(argv[1], settings); + MergeTreePartChecker::checkDataPart(argv[1], settings, Block()); } catch (...) { - DB::tryLogCurrentException(__PRETTY_FUNCTION__); + tryLogCurrentException(__PRETTY_FUNCTION__); throw; } From b06bdb0edf5569ddc239423745465d1b3c18ddc7 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 28 May 2015 15:32:43 +0300 Subject: [PATCH 038/109] dbms: add SSE variants of lower/upper and UTF8 equivalents [#METR-14764] --- dbms/include/DB/Common/PODArray.h | 2 + dbms/include/DB/Functions/FunctionsString.h | 176 ++++++++++++++++++++ dbms/src/Functions/FunctionsString.cpp | 4 + 3 files changed, 182 insertions(+) diff --git a/dbms/include/DB/Common/PODArray.h b/dbms/include/DB/Common/PODArray.h index beffd667969..7d96889483e 100644 --- a/dbms/include/DB/Common/PODArray.h +++ b/dbms/include/DB/Common/PODArray.h @@ -192,6 +192,8 @@ public: return *this; } + T * data() { return t_start(); } + const T * data() const { return t_start(); } size_t size() const { return t_end() - t_start(); } bool empty() const { return t_end() == t_start(); } diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index 14ea47f5478..ace57112981 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -234,6 +234,172 @@ private: } }; +template +struct LowerUpperImplVectorized +{ + template friend class LowerUpperUTF8ImplVectorized; + + static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets, + ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets) + { + res_data.resize(data.size()); + res_offsets.assign(offsets); + array(data.data(), data.data() + data.size(), res_data.data()); + } + + static void vector_fixed(const ColumnString::Chars_t & data, size_t n, + ColumnString::Chars_t & res_data) + { + res_data.resize(data.size()); + array(data.data(), data.data() + data.size(), res_data.data()); + } + + static void constant(const std::string & data, std::string & res_data) + { + res_data.resize(data.size()); + array(reinterpret_cast(data.data()), reinterpret_cast(data.data() + data.size()), + reinterpret_cast(&res_data[0])); + } + +private: + static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) + { + const auto src_end_sse = src_end - (src_end - src) % 16; + + const auto flip_case_mask = 1 << 5; + + const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); + const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); + const auto v_flip_case_mask = _mm_set1_epi8(flip_case_mask); + + for (; src < src_end_sse; src += 16, dst += 16) + { + /// load 16 sequential 8-bit characters + const auto chars = _mm_loadu_si128(reinterpret_cast(src)); + + /// find which 8-bit sequences belong to range [case_lower_bound, case_upper_bound] + const auto is_not_case = _mm_and_si128(_mm_cmpgt_epi8(chars, v_not_case_lower_bound), + _mm_cmplt_epi8(chars, v_not_case_upper_bound)); + + /// keep `flip_case_mask` only where necessary, zero out elsewhere + const auto xor_mask = _mm_and_si128(v_flip_case_mask, is_not_case); + + /// flip case by applying calculated mask + const auto cased_chars = _mm_xor_si128(chars, xor_mask); + + /// store result back to destination + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), cased_chars); + } + + for (; src < src_end; ++src, ++dst) + *dst = (*src >= not_case_lower_bound && *src <= not_case_upper_bound) ? *src ^ flip_case_mask : *src; + } +}; + +template +struct LowerUpperUTF8ImplVectorized +{ + static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets, + ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets) + { + res_data.resize(data.size()); + res_offsets.assign(offsets); + array(data.data(), data.data() + data.size(), res_data.data()); + } + + static void vector_fixed(const ColumnString::Chars_t & data, size_t n, + ColumnString::Chars_t & res_data) + { + res_data.resize(data.size()); + array(data.data(), data.data() + data.size(), res_data.data()); + } + + static void constant(const std::string & data, std::string & res_data) + { + res_data.resize(data.size()); + array(reinterpret_cast(data.data()), reinterpret_cast(data.data() + data.size()), + reinterpret_cast(&res_data[0])); + } + +private: + static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) + { + auto is_ascii = false; + + if (isCaseASCII(src, src_end, is_ascii)) + std::copy(src, src_end, dst); + else if (is_ascii) + LowerUpperImplVectorized::array(src, src_end, dst); + else + UTF8ToCase(src, src_end, dst); + } + + static bool isCaseASCII(const UInt8 * src, const UInt8 * const src_end, bool & is_ascii) + { + const auto src_end_sse = src_end - (src_end - src) % 16; + + const auto not_case_a_16 = _mm_set1_epi8('A' - 1); + const auto not_case_z_16 = _mm_set1_epi8('Z' + 1); + const auto zero_16 = _mm_setzero_si128(); + + auto is_case = true; + + for (; src < src_end_sse; src += 16) + { + const auto chars = _mm_loadu_si128(reinterpret_cast(src)); + + /// check for ASCII and case + const auto is_not_ascii = _mm_cmplt_epi8(chars, zero_16); + const auto mask_is_not_ascii = _mm_movemask_epi8(is_not_ascii); + + if (mask_is_not_ascii != 0) + { + is_ascii = false; + return false; + } + + const auto is_not_case = _mm_and_si128(_mm_cmpgt_epi8(chars, not_case_a_16), + _mm_cmplt_epi8(chars, not_case_z_16)); + const auto mask_is_not_case = _mm_movemask_epi8(is_not_case); + + if (mask_is_not_case != 0) + is_case = false; + } + + /// handle remaining symbols + for (; src < src_end; ++src) + if (*src > '\x7f') + { + is_ascii = false; + return false; + } + else if (*src >= 'A' && *src <= 'Z') + is_case = false; + + is_ascii = true; + return is_case; + } + + static void UTF8ToCase(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) + { + static const Poco::UTF8Encoding utf8; + + while (src < src_end) + { + if (const auto chars = utf8.convert(to_case(utf8.convert(src)), dst, src_end - src)) + { + src += chars; + dst += chars; + } + else + { + ++src; + ++dst; + } + } + } +}; + /** Если строка содержит текст в кодировке UTF-8 - перевести его в нижний (верхний) регистр. * Замечание: предполагается, что после перевода символа в другой регистр, @@ -1424,6 +1590,11 @@ struct NameReverseUTF8 { static constexpr auto name = "reverseUTF8"; }; struct NameSubstring { static constexpr auto name = "substring"; }; struct NameSubstringUTF8 { static constexpr auto name = "substringUTF8"; }; +struct NameSSELower { static constexpr auto name = "sse_lower"; }; +struct NameSSEUpper { static constexpr auto name = "sse_upper"; }; +struct NameSSELowerUTF8 { static constexpr auto name = "sse_lowerUTF8"; }; +struct NameSSEUpperUTF8 { static constexpr auto name = "sse_upperUTF8"; }; + typedef FunctionStringOrArrayToT, NameEmpty, UInt8> FunctionEmpty; typedef FunctionStringOrArrayToT, NameNotEmpty, UInt8> FunctionNotEmpty; typedef FunctionStringOrArrayToT FunctionLength; @@ -1437,5 +1608,10 @@ typedef FunctionStringToString FunctionReve typedef FunctionStringNumNumToString FunctionSubstring; typedef FunctionStringNumNumToString FunctionSubstringUTF8; +using FunctionSSELower = FunctionStringToString, NameSSELower>; +using FunctionSSEUpper = FunctionStringToString, NameSSEUpper>; +using FunctionSSELowerUTF8 = FunctionStringToString, NameSSELowerUTF8>; +using FunctionSSEUpperUTF8 = FunctionStringToString, NameSSEUpperUTF8>; + } diff --git a/dbms/src/Functions/FunctionsString.cpp b/dbms/src/Functions/FunctionsString.cpp index 17cda08cbc3..eadd6bc6884 100644 --- a/dbms/src/Functions/FunctionsString.cpp +++ b/dbms/src/Functions/FunctionsString.cpp @@ -20,6 +20,10 @@ void registerFunctionsString(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); + factory.registerFunction(); } } From 79bad96d9824bbc916d754d464f6d0c9d4e54d4a Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Thu, 28 May 2015 15:58:09 +0300 Subject: [PATCH 039/109] dbms: Server: Performance improvements. [#METR-15210] --- dbms/include/DB/Functions/FunctionsRound.h | 52 +++++++++++++++++----- 1 file changed, 40 insertions(+), 12 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index 2a2ebd86307..1b527d3a39e 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -511,9 +511,16 @@ namespace DB static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { auto divisor = Op::prepare(scale); - size_t size = in.size(); - for (size_t i = 0; i < size; ++i) - out[i] = Op::compute(in[i], divisor); + + auto begin_in = &in[0]; + auto end_in = begin_in + in.size(); + auto p_out = &out[0]; + + for (auto p_in = begin_in; p_in != end_in; ++p_in) + { + *p_out = Op::compute(*p_in, divisor); + ++p_out; + } } static inline T apply(T val, size_t scale) @@ -540,24 +547,45 @@ namespace DB Scale mm_scale; Op::prepareScale(scale, mm_scale); - const size_t size = in.size(); const size_t data_count = std::tuple_size(); - size_t i; - for (i = 0; i < (size - data_count + 1); i += data_count) - Op::compute(reinterpret_cast(&in[i]), mm_scale, reinterpret_cast(&out[i])); + auto begin_in = &in[0]; + auto end_in = begin_in + in.size(); + auto limit = end_in - (data_count - 1); - if (i < size) + auto begin_out = &out[0]; + auto end_out = begin_out + out.size(); + + auto p_in = begin_in; + auto p_out = begin_out; + for (; p_in < limit; p_in += data_count) + { + Op::compute(reinterpret_cast(p_in), mm_scale, reinterpret_cast(p_out)); + p_out += data_count; + } + + if (p_in < end_in) { Data tmp{0}; - for (size_t j = 0; (j < data_count) && ((i + j) < size); ++j) - tmp[j] = in[i + j]; + auto begin_tmp = &tmp[0]; + auto end_tmp = begin_tmp + data_count; + + for (auto p_tmp = begin_tmp; (p_tmp != end_tmp) && (p_in != end_in); ++p_tmp) + { + *p_tmp = *p_in; + ++p_in; + } Data res; Op::compute(reinterpret_cast(&tmp), mm_scale, reinterpret_cast(&res)); - for (size_t j = 0; (j < data_count) && ((i + j) < size); ++j) - out[i + j] = res[j]; + auto begin_res = &res[0]; + auto end_res = begin_res + data_count; + for (auto p_res = begin_res; (p_res != end_res) && (p_out != end_out); ++p_res) + { + *p_out = *p_res; + ++p_out; + } } } From 8125cad146d111248cd2135fb36d60981beaf90e Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Thu, 28 May 2015 17:00:52 +0300 Subject: [PATCH 040/109] dbms: Server: Performance improvements. [#METR-15210] --- dbms/include/DB/Functions/FunctionsRound.h | 66 ++++---- .../00161_rounding_functions.reference | 152 +++++++++--------- 2 files changed, 110 insertions(+), 108 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index 1b527d3a39e..ef9a1ed4d34 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -295,13 +295,13 @@ namespace DB struct FloatRoundingComputation : public BaseFloatRoundingComputation { - static inline void prepareScale(size_t scale, Scale & mm_scale) + static inline void prepare(size_t scale, Scale & mm_scale) { Float32 fscale = static_cast(scale); mm_scale = _mm_load1_ps(&fscale); } - static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) + static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) { __m128 val = _mm_loadu_ps(in); val = _mm_mul_ps(val, scale); @@ -315,13 +315,13 @@ namespace DB struct FloatRoundingComputation : public BaseFloatRoundingComputation { - static inline void prepareScale(size_t scale, Scale & mm_scale) + static inline void prepare(size_t scale, Scale & mm_scale) { Float32 fscale = static_cast(scale); mm_scale = _mm_load1_ps(&fscale); } - static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) + static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) { __m128 val = _mm_loadu_ps(in); @@ -379,11 +379,11 @@ namespace DB struct FloatRoundingComputation : public BaseFloatRoundingComputation { - static inline void prepareScale(size_t scale, Scale & mm_scale) + static inline void prepare(size_t scale, Scale & mm_scale) { } - static inline void compute(const Float32 * in, const Scale & scale, Float32 * out) + static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) { __m128 val = _mm_loadu_ps(in); val = _mm_round_ps(val, rounding_mode); @@ -395,13 +395,13 @@ namespace DB struct FloatRoundingComputation : public BaseFloatRoundingComputation { - static inline void prepareScale(size_t scale, Scale & mm_scale) + static inline void prepare(size_t scale, Scale & mm_scale) { Float64 fscale = static_cast(scale); mm_scale = _mm_load1_pd(&fscale); } - static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) + static inline void compute(const Float64 * __restrict in, const Scale & scale, Float64 * __restrict out) { __m128d val = _mm_loadu_pd(in); val = _mm_mul_pd(val, scale); @@ -415,13 +415,13 @@ namespace DB struct FloatRoundingComputation : public BaseFloatRoundingComputation { - static inline void prepareScale(size_t scale, Scale & mm_scale) + static inline void prepare(size_t scale, Scale & mm_scale) { Float64 fscale = static_cast(scale); mm_scale = _mm_load1_pd(&fscale); } - static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) + static inline void compute(const Float64 * __restrict in, const Scale & scale, Float64 * __restrict out) { __m128d val = _mm_loadu_pd(in); @@ -479,11 +479,11 @@ namespace DB struct FloatRoundingComputation : public BaseFloatRoundingComputation { - static inline void prepareScale(size_t scale, Scale & mm_scale) + static inline void prepare(size_t scale, Scale & mm_scale) { } - static inline void compute(const Float64 * in, const Scale & scale, Float64 * out) + static inline void compute(const Float64 * __restrict in, const Scale & scale, Float64 * __restrict out) { __m128d val = _mm_loadu_pd(in); val = _mm_round_pd(val, rounding_mode); @@ -512,11 +512,11 @@ namespace DB { auto divisor = Op::prepare(scale); - auto begin_in = &in[0]; - auto end_in = begin_in + in.size(); - auto p_out = &out[0]; + const T* begin_in = &in[0]; + const T* end_in = begin_in + in.size(); - for (auto p_in = begin_in; p_in != end_in; ++p_in) + T* __restrict p_out = &out[0]; + for (const T* __restrict p_in = begin_in; p_in != end_in; ++p_in) { *p_out = Op::compute(*p_in, divisor); ++p_out; @@ -545,43 +545,45 @@ namespace DB static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { Scale mm_scale; - Op::prepareScale(scale, mm_scale); + Op::prepare(scale, mm_scale); const size_t data_count = std::tuple_size(); - auto begin_in = &in[0]; - auto end_in = begin_in + in.size(); - auto limit = end_in - (data_count - 1); + const T* begin_in = &in[0]; + const T* end_in = begin_in + in.size(); - auto begin_out = &out[0]; - auto end_out = begin_out + out.size(); + T* begin_out = &out[0]; + const T* end_out = begin_out + out.size(); - auto p_in = begin_in; - auto p_out = begin_out; + const T* limit = end_in - (data_count - 1); + + const T* __restrict p_in = begin_in; + T* __restrict p_out = begin_out; for (; p_in < limit; p_in += data_count) { - Op::compute(reinterpret_cast(p_in), mm_scale, reinterpret_cast(p_out)); + Op::compute(p_in, mm_scale, p_out); p_out += data_count; } if (p_in < end_in) { Data tmp{0}; - auto begin_tmp = &tmp[0]; - auto end_tmp = begin_tmp + data_count; + T* begin_tmp = &tmp[0]; + const T* end_tmp = begin_tmp + data_count; - for (auto p_tmp = begin_tmp; (p_tmp != end_tmp) && (p_in != end_in); ++p_tmp) + for (T* __restrict p_tmp = begin_tmp; (p_tmp != end_tmp) && (p_in != end_in); ++p_tmp) { *p_tmp = *p_in; ++p_in; } Data res; + const T* begin_res = &res[0]; + const T* end_res = begin_res + data_count; + Op::compute(reinterpret_cast(&tmp), mm_scale, reinterpret_cast(&res)); - auto begin_res = &res[0]; - auto end_res = begin_res + data_count; - for (auto p_res = begin_res; (p_res != end_res) && (p_out != end_out); ++p_res) + for (const T* __restrict p_res = begin_res; (p_res != end_res) && (p_out != end_out); ++p_res) { *p_out = *p_res; ++p_out; @@ -596,7 +598,7 @@ namespace DB else { Scale mm_scale; - Op::prepareScale(scale, mm_scale); + Op::prepare(scale, mm_scale); Data tmp{0}; tmp[0] = val; diff --git a/dbms/tests/queries/0_stateless/00161_rounding_functions.reference b/dbms/tests/queries/0_stateless/00161_rounding_functions.reference index 990df3ca74b..9b1be1333d2 100644 --- a/dbms/tests/queries/0_stateless/00161_rounding_functions.reference +++ b/dbms/tests/queries/0_stateless/00161_rounding_functions.reference @@ -458,16 +458,16 @@ -13 -13 -13 -13 -13 -13 -13 -13 -13 --16 -6 -16 --16 -6 -16 --16 -6 -16 --16 -6 -16 --13 -6 -16 --13 -6 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-13 -20 -10 +-13 -20 -10 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 0 0 0 0 0 0 -13 -13 -13 @@ -499,18 +499,18 @@ -13 -13 -13 -13 -13 -13 -13 -13 -13 --16 -6 -16 --16 -6 -16 --16 -6 -16 --16 -6 -16 --13 -6 -16 --13 -6 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-13 -20 -10 +-13 -20 -10 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 -13 -13 -13 -13 -13 -13 -13 -13 -13 @@ -540,18 +540,18 @@ -13 -13 -13 -13 -13 -13 -13 -13 -13 --16 -6 -16 --16 -6 -16 --16 -6 -16 --16 -6 -16 --13 -6 -16 --13 -6 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-13 -20 -10 +-13 -20 -10 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 -13 -13 -13 -13 -13 -13 -13 -13 -13 @@ -581,18 +581,18 @@ -13 -13 -13 -13 -13 -13 -13 -13 -13 --16 -6 -16 --16 -6 -16 --16 -6 -16 --16 -6 -16 --13 -6 -16 --13 -6 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 --16 84 -16 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-13 -20 -10 +-13 -20 -10 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 -13 -13 -13 -13 -13 -13 -13 -13 -13 @@ -622,18 +622,18 @@ -13 -13 -13 -13 -13 -13 -13 -13 -13 -0 0 0 -0 0 0 -0 0 0 -0 0 0 --13 0 0 --13 0 0 -0 0 0 -0 0 0 -0 0 0 -0 0 0 -0 0 0 -0 0 0 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-13 -20 -10 +-13 -20 -10 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 -13 -13 -13 -13 -13 -13 -13 -13 -13 @@ -663,18 +663,18 @@ -13 -13 -13 -13 -13 -13 -13 -13 -13 -0 0 0 -0 0 0 -0 0 0 -0 0 0 --13 0 0 --13 0 0 -0 0 0 -0 0 0 -0 0 0 -0 0 0 -0 0 0 -0 0 0 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-10 -20 -10 +-13 -20 -10 +-13 -20 -10 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 +0 -100 0 2.72 2.72 2.71 2.72 2.72 2.71 2.72 2.72 2.71 @@ -745,12 +745,12 @@ -3 -2 -3 -3 -2 -3 -3 -2 -3 -0 0 0 -0 0 0 -0 0 0 -0 0 0 --2.7 0 0 --2.7 0 0 +0 -10 0 +0 -10 0 +0 -10 0 +0 -10 0 +-2.7 -10 0 +-2.7 -10 0 0 0 0 0 0 0 0 0 0 From 3786513a3c33676cd838eed897220fef34cb4c6f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2015 00:41:28 +0300 Subject: [PATCH 041/109] dbms: fixing error [#METR-16575]. --- dbms/include/DB/Client/Connection.h | 61 ++++++++++++++----- dbms/include/DB/Client/ConnectionPool.h | 27 +++++++- dbms/include/DB/Interpreters/Cluster.h | 4 +- .../DB/TableFunctions/TableFunctionRemote.h | 10 +-- dbms/src/Client/Connection.cpp | 6 +- dbms/src/Interpreters/Cluster.cpp | 34 +++++++---- dbms/src/Storages/StorageSystemClusters.cpp | 4 +- 7 files changed, 103 insertions(+), 43 deletions(-) diff --git a/dbms/include/DB/Client/Connection.h b/dbms/include/DB/Client/Connection.h index 2d40c2e6141..04605932b65 100644 --- a/dbms/include/DB/Client/Connection.h +++ b/dbms/include/DB/Client/Connection.h @@ -56,12 +56,38 @@ public: Poco::Timespan ping_timeout_ = Poco::Timespan(DBMS_DEFAULT_PING_TIMEOUT_SEC, 0)) : host(host_), port(port_), default_database(default_database_), - user(user_), password(password_), + user(user_), password(password_), resolved_address(host, port), client_name(client_name_), compression(compression_), connect_timeout(connect_timeout_), receive_timeout(receive_timeout_), send_timeout(send_timeout_), ping_timeout(ping_timeout_), - log_wrapper(host, port) + log_wrapper(*this) + { + /// Соединеняемся не сразу, а при первой необходимости. + + if (user.empty()) + user = "default"; + } + + Connection(const String & host_, UInt16 port_, const Poco::Net::SocketAddress & resolved_address_, + const String & default_database_, + const String & user_, const String & password_, + const String & client_name_ = "client", + Protocol::Compression::Enum compression_ = Protocol::Compression::Enable, + Poco::Timespan connect_timeout_ = Poco::Timespan(DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, 0), + Poco::Timespan receive_timeout_ = Poco::Timespan(DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, 0), + Poco::Timespan send_timeout_ = Poco::Timespan(DBMS_DEFAULT_SEND_TIMEOUT_SEC, 0), + Poco::Timespan ping_timeout_ = Poco::Timespan(DBMS_DEFAULT_PING_TIMEOUT_SEC, 0)) + : + host(host_), port(port_), + default_database(default_database_), + user(user_), password(password_), + resolved_address(resolved_address_), + client_name(client_name_), + compression(compression_), + connect_timeout(connect_timeout_), receive_timeout(receive_timeout_), send_timeout(send_timeout_), + ping_timeout(ping_timeout_), + log_wrapper(*this) { /// Соединеняемся не сразу, а при первой необходимости. @@ -99,6 +125,16 @@ public: /// Адрес сервера - для сообщений в логе и в эксепшенах. String getServerAddress() const; + const String & getHost() const + { + return host; + } + + UInt16 getPort() const + { + return port; + } + /// Если последний флаг true, то затем необходимо вызвать sendExternalTablesData void sendQuery(const String & query, const String & query_id_ = "", UInt64 stage = QueryProcessingStage::Complete, const Settings * settings = nullptr, bool with_pending_data = false); @@ -131,16 +167,6 @@ public: */ void disconnect(); - const std::string & getHost() const - { - return host; - } - - UInt16 getPort() const - { - return port; - } - size_t outBytesCount() const { return !out.isNull() ? out->count() : 0; } size_t inBytesCount() const { return !in.isNull() ? in->count() : 0; } @@ -151,6 +177,9 @@ private: String user; String password; + /// Адрес может быть заранее отрезолвен и передан в конструктор. Тогда поля host и port имеют смысл только для логгирования. + Poco::Net::SocketAddress resolved_address; + String client_name; bool connected = false; @@ -191,22 +220,22 @@ private: class LoggerWrapper { public: - LoggerWrapper(std::string & host_, size_t port_) : log(nullptr), host(host_), port(port_) + LoggerWrapper(Connection & parent_) + : log(nullptr), parent(parent_) { } Logger * get() { if (!log) - log = &Logger::get("Connection (" + Poco::Net::SocketAddress(host, port).toString() + ")"); + log = &Logger::get("Connection (" + parent.getServerAddress() + ")"); return log; } private: std::atomic log; - std::string host; - size_t port; + Connection & parent; }; LoggerWrapper log_wrapper; diff --git a/dbms/include/DB/Client/ConnectionPool.h b/dbms/include/DB/Client/ConnectionPool.h index aa779815bf7..d8908096f89 100644 --- a/dbms/include/DB/Client/ConnectionPool.h +++ b/dbms/include/DB/Client/ConnectionPool.h @@ -54,16 +54,34 @@ public: typedef PoolBase Base; ConnectionPool(unsigned max_connections_, - const String & host_, UInt16 port_, const String & default_database_, + const String & host_, UInt16 port_, + const String & default_database_, const String & user_, const String & password_, const String & client_name_ = "client", Protocol::Compression::Enum compression_ = Protocol::Compression::Enable, Poco::Timespan connect_timeout_ = Poco::Timespan(DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, 0), Poco::Timespan receive_timeout_ = Poco::Timespan(DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, 0), Poco::Timespan send_timeout_ = Poco::Timespan(DBMS_DEFAULT_SEND_TIMEOUT_SEC, 0)) - : Base(max_connections_, &Logger::get("ConnectionPool (" + Poco::Net::SocketAddress(host_, port_).toString() + ")")), + : Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")), host(host_), port(port_), default_database(default_database_), - user(user_), password(password_), + user(user_), password(password_), resolved_address(host_, port_), + client_name(client_name_), compression(compression_), + connect_timeout(connect_timeout_), receive_timeout(receive_timeout_), send_timeout(send_timeout_) + { + } + + ConnectionPool(unsigned max_connections_, + const String & host_, UInt16 port_, const Poco::Net::SocketAddress & resolved_address_, + const String & default_database_, + const String & user_, const String & password_, + const String & client_name_ = "client", + Protocol::Compression::Enum compression_ = Protocol::Compression::Enable, + Poco::Timespan connect_timeout_ = Poco::Timespan(DBMS_DEFAULT_CONNECT_TIMEOUT_SEC, 0), + Poco::Timespan receive_timeout_ = Poco::Timespan(DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC, 0), + Poco::Timespan send_timeout_ = Poco::Timespan(DBMS_DEFAULT_SEND_TIMEOUT_SEC, 0)) + : Base(max_connections_, &Logger::get("ConnectionPool (" + host_ + ":" + toString(port_) + ")")), + host(host_), port(port_), default_database(default_database_), + user(user_), password(password_), resolved_address(resolved_address_), client_name(client_name_), compression(compression_), connect_timeout(connect_timeout_), receive_timeout(receive_timeout_), send_timeout(send_timeout_) { @@ -101,6 +119,9 @@ private: String user; String password; + /// Адрес может быть заранее отрезолвен и передан в конструктор. Тогда поля host и port имеют смысл только для логгирования. + Poco::Net::SocketAddress resolved_address; + String client_name; Protocol::Compression::Enum compression; /// Сжимать ли данные при взаимодействии с сервером. diff --git a/dbms/include/DB/Interpreters/Cluster.h b/dbms/include/DB/Interpreters/Cluster.h index 0bd30781269..fc5b0410371 100644 --- a/dbms/include/DB/Interpreters/Cluster.h +++ b/dbms/include/DB/Interpreters/Cluster.h @@ -8,6 +8,7 @@ namespace DB { + /// Cluster содержит пулы соединений до каждого из узлов /// С локальными узлами соединение не устанавливается, а выполяется запрос напрямую. /// Поэтому храним только количество локальных узлов @@ -61,8 +62,9 @@ public: * * */ - Poco::Net::SocketAddress host_port; + Poco::Net::SocketAddress resolved_address; String host_name; + UInt16 port; String user; String password; UInt32 replica_num; diff --git a/dbms/include/DB/TableFunctions/TableFunctionRemote.h b/dbms/include/DB/TableFunctions/TableFunctionRemote.h index f407d70027f..47e4f3f5892 100644 --- a/dbms/include/DB/TableFunctions/TableFunctionRemote.h +++ b/dbms/include/DB/TableFunctions/TableFunctionRemote.h @@ -197,7 +197,7 @@ private: return true; } - /* Парсит строку, генерирующую шарды и реплики. Splitter - один из двух символов | или ' + /* Парсит строку, генерирующую шарды и реплики. Разделитель - один из двух символов | или , * в зависимости от того генерируются шарды или реплики. * Например: * host1,host2,... - порождает множество шардов из host1, host2, ... @@ -209,7 +209,7 @@ private: * abc{1..9}de{f,g,h} - прямое произведение, 27 шардов. * abc{1..9}de{0|1} - прямое произведение, 9 шардов, в каждом 2 реплики. */ - std::vector parseDescription(const String & description, size_t l, size_t r, char splitter) const + std::vector parseDescription(const String & description, size_t l, size_t r, char separator) const { std::vector res; std::vector cur; @@ -238,7 +238,7 @@ private: if (description[m] == '{') ++cnt; if (description[m] == '}') --cnt; if (description[m] == '.' && description[m-1] == '.') last_dot = m; - if (description[m] == splitter) have_splitter = true; + if (description[m] == separator) have_splitter = true; if (cnt == 0) break; } if (cnt != 0) @@ -282,13 +282,13 @@ private: buffer.push_back(cur); } } else if (have_splitter) /// Если внутри есть текущий разделитель, то сгенерировать множество получаемых строк - buffer = parseDescription(description, i + 1, m, splitter); + buffer = parseDescription(description, i + 1, m, separator); else /// Иначе просто скопировать, порождение произойдет при вызове с правильным разделителем buffer.push_back(description.substr(i, m - i + 1)); /// К текущему множеству строк добавить все возможные полученные продолжения append(cur, buffer); i = m; - } else if (description[i] == splitter) { + } else if (description[i] == separator) { /// Если разделитель, то добавляем в ответ найденные строки res.insert(res.end(), cur.begin(), cur.end()); cur.clear(); diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 9e22eda4225..0eeb6247997 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -34,9 +34,9 @@ void Connection::connect() if (connected) disconnect(); - LOG_TRACE(log_wrapper.get(), "Connecting to " << default_database << "@" << host << ":" << port); + LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << default_database << ". User: " << user); - socket.connect(Poco::Net::SocketAddress(host, port), connect_timeout); + socket.connect(resolved_address, connect_timeout); socket.setReceiveTimeout(receive_timeout); socket.setSendTimeout(send_timeout); socket.setNoDelay(true); @@ -501,7 +501,7 @@ void Connection::initBlockInput() String Connection::getServerAddress() const { - return Poco::Net::SocketAddress(host, port).toString(); + return host + ":" + toString(resolved_address.port()) + ", " + resolved_address.host().toString(); } diff --git a/dbms/src/Interpreters/Cluster.cpp b/dbms/src/Interpreters/Cluster.cpp index 9b71d0a7cdf..7f39c715a00 100644 --- a/dbms/src/Interpreters/Cluster.cpp +++ b/dbms/src/Interpreters/Cluster.cpp @@ -13,11 +13,8 @@ Cluster::Address::Address(const String & config_prefix) auto & config = Poco::Util::Application::instance().config(); host_name = config.getString(config_prefix + ".host"); - host_port = Poco::Net::SocketAddress( - host_name, - config.getInt(config_prefix + ".port") - ); - + port = config.getInt(config_prefix + ".port"); + resolved_address = Poco::Net::SocketAddress(host_name, port); user = config.getString(config_prefix + ".user", "default"); password = config.getString(config_prefix + ".password", ""); } @@ -29,9 +26,17 @@ Cluster::Address::Address(const String & host_port_, const String & user_, const /// Похоже на то, что строка host_port_ содержит порт. Если условие срабатывает - не обязательно значит, что порт есть (пример: [::]). if (nullptr != strchr(host_port_.c_str(), ':') || !default_port) - host_port = Poco::Net::SocketAddress(host_port_); + { + resolved_address = Poco::Net::SocketAddress(host_port_); + host_name = host_port_.substr(0, host_port_.find(':')); + port = resolved_address.port(); + } else - host_port = Poco::Net::SocketAddress(host_port_, default_port); + { + resolved_address = Poco::Net::SocketAddress(host_port_, default_port); + host_name = host_port_; + port = default_port; + } } namespace @@ -41,8 +46,8 @@ namespace return escapeForFileName(address.user) + (address.password.empty() ? "" : (':' + escapeForFileName(address.password))) + '@' + - escapeForFileName(address.host_port.host().toString()) + ':' + - std::to_string(address.host_port.port()); + escapeForFileName(address.resolved_address.host().toString()) + ':' + + std::to_string(address.resolved_address.port()); } } @@ -178,7 +183,8 @@ Cluster::Cluster(const Settings & settings, const String & cluster_name) { replicas.emplace_back(new ConnectionPool( settings.distributed_connections_pool_size, - replica.host_port.host().toString(), replica.host_port.port(), "", replica.user, replica.password, + replica.host_name, replica.port, replica.resolved_address, + "", replica.user, replica.password, "server", Protocol::Compression::Enable, saturate(settings.connect_timeout_with_failover_ms, settings.limits.max_execution_time), saturate(settings.receive_timeout, settings.limits.max_execution_time), @@ -204,7 +210,8 @@ Cluster::Cluster(const Settings & settings, const String & cluster_name) { pools.emplace_back(new ConnectionPool( settings.distributed_connections_pool_size, - address.host_port.host().toString(), address.host_port.port(), "", address.user, address.password, + address.host_name, address.port, address.resolved_address, + "", address.user, address.password, "server", Protocol::Compression::Enable, saturate(settings.connect_timeout, settings.limits.max_execution_time), saturate(settings.receive_timeout, settings.limits.max_execution_time), @@ -237,7 +244,8 @@ Cluster::Cluster(const Settings & settings, std::vector> nam { replicas.emplace_back(new ConnectionPool( settings.distributed_connections_pool_size, - replica.host_port.host().toString(), replica.host_port.port(), "", replica.user, replica.password, + replica.host_name, replica.port, replica.resolved_address, + "", replica.user, replica.password, "server", Protocol::Compression::Enable, saturate(settings.connect_timeout_with_failover_ms, settings.limits.max_execution_time), saturate(settings.receive_timeout, settings.limits.max_execution_time), @@ -264,7 +272,7 @@ bool Cluster::isLocal(const Address & address) /// - её порт совпадает с портом, который слушает сервер; /// - её хост резолвится в набор адресов, один из которых совпадает с одним из адресов сетевых интерфейсов сервера /// то нужно всегда ходить на этот шард без межпроцессного взаимодействия - return isLocalAddress(address.host_port); + return isLocalAddress(address.resolved_address); } } diff --git a/dbms/src/Storages/StorageSystemClusters.cpp b/dbms/src/Storages/StorageSystemClusters.cpp index 37e99e9daed..be9bf3500ad 100644 --- a/dbms/src/Storages/StorageSystemClusters.cpp +++ b/dbms/src/Storages/StorageSystemClusters.cpp @@ -62,8 +62,8 @@ BlockInputStreams StorageSystemClusters::read( replica_num_column->insert(static_cast(address.replica_num)); host_name_column->insert(address.host_name); - host_address_column->insert(address.host_port.host().toString()); - port_column->insert(static_cast(address.host_port.port())); + host_address_column->insert(address.resolved_address.host().toString()); + port_column->insert(static_cast(address.port)); user_column->insert(address.user); }; From 3db1bf63a2c11d320921b485dc2662880e941403 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2015 03:33:56 +0300 Subject: [PATCH 042/109] dbms: fixed error [#METR-16575] [#METR-16573]. --- dbms/include/DB/Client/Connection.h | 21 +++++++-- dbms/include/DB/Client/ConnectionPool.h | 7 ++- .../DB/Client/ConnectionPoolWithFailover.h | 3 +- dbms/include/DB/Common/getFQDNOrHostName.h | 8 ++++ .../DB/Functions/FunctionsMiscellaneous.h | 2 + dbms/src/Client/Connection.cpp | 46 ++++++++++--------- dbms/src/Client/ParallelReplicas.cpp | 2 +- dbms/src/Common/getFQDNOrHostName.cpp | 25 ++++++++++ dbms/src/Server/Server.cpp | 8 +++- 9 files changed, 92 insertions(+), 30 deletions(-) create mode 100644 dbms/include/DB/Common/getFQDNOrHostName.h create mode 100644 dbms/src/Common/getFQDNOrHostName.cpp diff --git a/dbms/include/DB/Client/Connection.h b/dbms/include/DB/Client/Connection.h index 04605932b65..38305049e58 100644 --- a/dbms/include/DB/Client/Connection.h +++ b/dbms/include/DB/Client/Connection.h @@ -67,6 +67,8 @@ public: if (user.empty()) user = "default"; + + setDescription(); } Connection(const String & host_, UInt16 port_, const Poco::Net::SocketAddress & resolved_address_, @@ -93,6 +95,8 @@ public: if (user.empty()) user = "default"; + + setDescription(); } virtual ~Connection() {}; @@ -122,8 +126,11 @@ public: void getServerVersion(String & name, UInt64 & version_major, UInt64 & version_minor, UInt64 & revision); - /// Адрес сервера - для сообщений в логе и в эксепшенах. - String getServerAddress() const; + /// Для сообщений в логе и в эксепшенах. + const String & getDescription() const + { + return description; + } const String & getHost() const { @@ -177,9 +184,15 @@ private: String user; String password; - /// Адрес может быть заранее отрезолвен и передан в конструктор. Тогда поля host и port имеют смысл только для логгирования. + /** Адрес может быть заранее отрезолвен и передан в конструктор. Тогда поля host и port имеют смысл только для логгирования. + * Иначе адрес резолвится в конструкторе. То есть, DNS балансировка не поддерживается. + */ Poco::Net::SocketAddress resolved_address; + /// Для сообщений в логе и в эксепшенах. + String description; + void setDescription(); + String client_name; bool connected = false; @@ -228,7 +241,7 @@ private: Logger * get() { if (!log) - log = &Logger::get("Connection (" + parent.getServerAddress() + ")"); + log = &Logger::get("Connection (" + parent.getDescription() + ")"); return log; } diff --git a/dbms/include/DB/Client/ConnectionPool.h b/dbms/include/DB/Client/ConnectionPool.h index d8908096f89..3e65e149a74 100644 --- a/dbms/include/DB/Client/ConnectionPool.h +++ b/dbms/include/DB/Client/ConnectionPool.h @@ -107,7 +107,8 @@ protected: ConnectionPtr allocObject() override { return new Connection( - host, port, default_database, user, password, + host, port, resolved_address, + default_database, user, password, client_name, compression, connect_timeout, receive_timeout, send_timeout); } @@ -119,7 +120,9 @@ private: String user; String password; - /// Адрес может быть заранее отрезолвен и передан в конструктор. Тогда поля host и port имеют смысл только для логгирования. + /** Адрес может быть заранее отрезолвен и передан в конструктор. Тогда поля host и port имеют смысл только для логгирования. + * Иначе адрес резолвится в конструкторе. То есть, DNS балансировка не поддерживается. + */ Poco::Net::SocketAddress resolved_address; String client_name; diff --git a/dbms/include/DB/Client/ConnectionPoolWithFailover.h b/dbms/include/DB/Client/ConnectionPoolWithFailover.h index ad61ad5c262..bded7ff3614 100644 --- a/dbms/include/DB/Client/ConnectionPoolWithFailover.h +++ b/dbms/include/DB/Client/ConnectionPoolWithFailover.h @@ -5,6 +5,7 @@ #include +#include #include @@ -33,7 +34,7 @@ public: : Base(nested_pools_, max_tries_, decrease_error_period_, &Logger::get("ConnectionPoolWithFailover")), default_load_balancing(load_balancing) { - std::string local_hostname = Poco::Net::DNS::hostName(); + const std::string & local_hostname = getFQDNOrHostName(); hostname_differences.resize(nested_pools.size()); for (size_t i = 0; i < nested_pools.size(); ++i) diff --git a/dbms/include/DB/Common/getFQDNOrHostName.h b/dbms/include/DB/Common/getFQDNOrHostName.h new file mode 100644 index 00000000000..7e1c1b43040 --- /dev/null +++ b/dbms/include/DB/Common/getFQDNOrHostName.h @@ -0,0 +1,8 @@ +#pragma once + +#include + +/** Получить FQDN для локального сервера путём DNS-резолвинга hostname - аналогично вызову утилиты hostname с флагом -f. + * Если не получилось отрезолвить, то вернуть hostname - аналогично вызову утилиты hostname без флагов или uname -n. + */ +const std::string & getFQDNOrHostName(); diff --git a/dbms/include/DB/Functions/FunctionsMiscellaneous.h b/dbms/include/DB/Functions/FunctionsMiscellaneous.h index 2cb9d314362..f3f67448e89 100644 --- a/dbms/include/DB/Functions/FunctionsMiscellaneous.h +++ b/dbms/include/DB/Functions/FunctionsMiscellaneous.h @@ -112,6 +112,7 @@ public: } }; + /// Получить имя хоста. (Оно - константа, вычисляется один раз за весь запрос.) class FunctionHostName : public IFunction { @@ -146,6 +147,7 @@ public: } }; + class FunctionVisibleWidth : public IFunction { public: diff --git a/dbms/src/Client/Connection.cpp b/dbms/src/Client/Connection.cpp index 0eeb6247997..4afed87c523 100644 --- a/dbms/src/Client/Connection.cpp +++ b/dbms/src/Client/Connection.cpp @@ -34,7 +34,7 @@ void Connection::connect() if (connected) disconnect(); - LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << default_database << ". User: " << user); + LOG_TRACE(log_wrapper.get(), "Connecting. Database: " << (default_database.empty() ? "(not specified)" : default_database) << ". User: " << user); socket.connect(resolved_address, connect_timeout); socket.setReceiveTimeout(receive_timeout); @@ -60,21 +60,21 @@ void Connection::connect() disconnect(); /// Добавляем в сообщение адрес сервера. Также объект Exception запомнит stack trace. Жаль, что более точный тип исключения теряется. - throw NetException(e.displayText(), "(" + getServerAddress() + ")", ErrorCodes::NETWORK_ERROR); + throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::NETWORK_ERROR); } catch (Poco::TimeoutException & e) { disconnect(); /// Добавляем в сообщение адрес сервера. Также объект Exception запомнит stack trace. Жаль, что более точный тип исключения теряется. - throw NetException(e.displayText(), "(" + getServerAddress() + ")", ErrorCodes::SOCKET_TIMEOUT); + throw NetException(e.displayText(), "(" + getDescription() + ")", ErrorCodes::SOCKET_TIMEOUT); } } void Connection::disconnect() { - //LOG_TRACE(log_wrapper.get(), "Disconnecting (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Disconnecting"); socket.close(); in = nullptr; @@ -85,7 +85,7 @@ void Connection::disconnect() void Connection::sendHello() { - //LOG_TRACE(log_wrapper.get(), "Sending hello (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Sending hello"); writeVarUInt(Protocol::Client::Hello, *out); writeStringBinary((DBMS_NAME " ") + client_name, *out); @@ -102,7 +102,7 @@ void Connection::sendHello() void Connection::receiveHello() { - //LOG_TRACE(log_wrapper.get(), "Receiving hello (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Receiving hello"); /// Получить hello пакет. UInt64 packet_type = 0; @@ -127,7 +127,7 @@ void Connection::receiveHello() /// Закроем соединение, чтобы не было рассинхронизации. disconnect(); - throw NetException("Unexpected packet from server " + getServerAddress() + " (expected Hello or Exception, got " + throw NetException("Unexpected packet from server " + getDescription() + " (expected Hello or Exception, got " + String(Protocol::Server::toString(packet_type)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } } @@ -192,7 +192,7 @@ struct PingTimeoutSetter bool Connection::ping() { - // LOG_TRACE(log_wrapper.get(), "Ping (" << getServerAddress() << ")"); + // LOG_TRACE(log_wrapper.get(), "Ping"); PingTimeoutSetter timeout_setter(socket, ping_timeout); try @@ -219,7 +219,7 @@ bool Connection::ping() if (pong != Protocol::Server::Pong) { - throw Exception("Unexpected packet from server " + getServerAddress() + " (expected Pong, got " + throw Exception("Unexpected packet from server " + getDescription() + " (expected Pong, got " + String(Protocol::Server::toString(pong)) + ")", ErrorCodes::UNEXPECTED_PACKET_FROM_SERVER); } @@ -242,7 +242,7 @@ void Connection::sendQuery(const String & query, const String & query_id_, UInt6 query_id = query_id_; - //LOG_TRACE(log_wrapper.get(), "Sending query (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Sending query"); writeVarUInt(Protocol::Client::Query, *out); @@ -281,7 +281,7 @@ void Connection::sendQuery(const String & query, const String & query_id_, UInt6 void Connection::sendCancel() { - //LOG_TRACE(log_wrapper.get(), "Sending cancel (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Sending cancel"); writeVarUInt(Protocol::Client::Cancel, *out); out->next(); @@ -290,7 +290,7 @@ void Connection::sendCancel() void Connection::sendData(const Block & block, const String & name) { - //LOG_TRACE(log_wrapper.get(), "Sending data (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Sending data"); if (!block_out) { @@ -405,7 +405,7 @@ bool Connection::hasReadBufferPendingData() const Connection::Packet Connection::receivePacket() { - //LOG_TRACE(log_wrapper.get(), "Receiving packet (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Receiving packet"); try { @@ -448,14 +448,14 @@ Connection::Packet Connection::receivePacket() disconnect(); throw Exception("Unknown packet " + toString(res.type) - + " from server " + getServerAddress(), ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); + + " from server " + getDescription(), ErrorCodes::UNKNOWN_PACKET_FROM_SERVER); } } catch (Exception & e) { /// Дописываем в текст исключения адрес сервера, если надо. if (e.code() != ErrorCodes::UNKNOWN_PACKET_FROM_SERVER) - e.addMessage("while receiving packet from " + getServerAddress()); + e.addMessage("while receiving packet from " + getDescription()); throw; } @@ -464,7 +464,7 @@ Connection::Packet Connection::receivePacket() Block Connection::receiveData() { - //LOG_TRACE(log_wrapper.get(), "Receiving data (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Receiving data"); initBlockInput(); @@ -499,25 +499,29 @@ void Connection::initBlockInput() } -String Connection::getServerAddress() const +void Connection::setDescription() { - return host + ":" + toString(resolved_address.port()) + ", " + resolved_address.host().toString(); + description = host + ":" + toString(resolved_address.port()); + auto ip_address = resolved_address.host().toString(); + + if (host != ip_address) + description += ", " + ip_address; } SharedPtr Connection::receiveException() { - //LOG_TRACE(log_wrapper.get(), "Receiving exception (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Receiving exception"); Exception e; - readException(e, *in, "Received from " + getServerAddress()); + readException(e, *in, "Received from " + getDescription()); return e.clone(); } Progress Connection::receiveProgress() { - //LOG_TRACE(log_wrapper.get(), "Receiving progress (" << getServerAddress() << ")"); + //LOG_TRACE(log_wrapper.get(), "Receiving progress"); Progress progress; progress.read(*in, server_revision); diff --git a/dbms/src/Client/ParallelReplicas.cpp b/dbms/src/Client/ParallelReplicas.cpp index ed0ab8cc657..81d629d392c 100644 --- a/dbms/src/Client/ParallelReplicas.cpp +++ b/dbms/src/Client/ParallelReplicas.cpp @@ -184,7 +184,7 @@ std::string ParallelReplicas::dumpAddresses() const const Connection * connection = e.second; if (connection != nullptr) { - os << (is_first ? "" : "; ") << connection->getServerAddress(); + os << (is_first ? "" : "; ") << connection->getDescription(); is_first = false; } } diff --git a/dbms/src/Common/getFQDNOrHostName.cpp b/dbms/src/Common/getFQDNOrHostName.cpp new file mode 100644 index 00000000000..9af9ca6e49d --- /dev/null +++ b/dbms/src/Common/getFQDNOrHostName.cpp @@ -0,0 +1,25 @@ +#include +#include + + +namespace +{ + std::string getFQDNOrHostNameImpl() + { + try + { + return Poco::Net::DNS::thisHost().name(); + } + catch (...) + { + return Poco::Net::DNS::hostName(); + } + } +} + + +const std::string & getFQDNOrHostName() +{ + static std::string result = getFQDNOrHostNameImpl(); + return result; +} diff --git a/dbms/src/Server/Server.cpp b/dbms/src/Server/Server.cpp index dd86f4d5116..09124ad946f 100644 --- a/dbms/src/Server/Server.cpp +++ b/dbms/src/Server/Server.cpp @@ -18,6 +18,7 @@ #include #include +#include #include #include #include @@ -485,9 +486,14 @@ int Server::main(const std::vector & args) { String this_host; if (config().has("interserver_http_host")) + { this_host = config().getString("interserver_http_host"); + } else - this_host = Poco::Net::DNS::hostName(); + { + this_host = getFQDNOrHostName(); + LOG_DEBUG(log, "Configuration parameter 'interserver_http_host' doesn't exist. Will use '" + this_host + "' as replica host."); + } String port_str = config().getString("interserver_http_port"); int port = parse(port_str); From 8825aa2db35d6737b75b521c38ccbbfb9fc08fcb Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 28 May 2015 18:34:37 +0300 Subject: [PATCH 043/109] dbms: SSE lower/upper(UTF8) fix hardcode and harness SSE4.2 [#METR-14764] --- dbms/include/DB/Functions/FunctionsString.h | 28 +++++++++++++-------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index ace57112981..8f50c95e420 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -14,6 +14,9 @@ #include #include +#include +#include + namespace DB { @@ -338,9 +341,10 @@ private: { const auto src_end_sse = src_end - (src_end - src) % 16; - const auto not_case_a_16 = _mm_set1_epi8('A' - 1); - const auto not_case_z_16 = _mm_set1_epi8('Z' + 1); - const auto zero_16 = _mm_setzero_si128(); +// const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); +// const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); + const auto v_not_case_range = _mm_set1_epi16((not_case_upper_bound << 8) | not_case_lower_bound); + const auto v_zero = _mm_setzero_si128(); auto is_case = true; @@ -349,7 +353,7 @@ private: const auto chars = _mm_loadu_si128(reinterpret_cast(src)); /// check for ASCII and case - const auto is_not_ascii = _mm_cmplt_epi8(chars, zero_16); + const auto is_not_ascii = _mm_cmplt_epi8(chars, v_zero); const auto mask_is_not_ascii = _mm_movemask_epi8(is_not_ascii); if (mask_is_not_ascii != 0) @@ -358,12 +362,16 @@ private: return false; } - const auto is_not_case = _mm_and_si128(_mm_cmpgt_epi8(chars, not_case_a_16), - _mm_cmplt_epi8(chars, not_case_z_16)); - const auto mask_is_not_case = _mm_movemask_epi8(is_not_case); - - if (mask_is_not_case != 0) + const auto is_case_result = _mm_cmpestra(v_not_case_range, 2, chars, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES); + if (is_case_result == 0) is_case = false; + +// const auto is_not_case = _mm_and_si128(_mm_cmpgt_epi8(chars, v_not_case_lower_bound), +// _mm_cmplt_epi8(chars, v_not_case_upper_bound)); +// const auto mask_is_not_case = _mm_movemask_epi8(is_not_case); +// +// if (mask_is_not_case != 0) +// is_case = false; } /// handle remaining symbols @@ -373,7 +381,7 @@ private: is_ascii = false; return false; } - else if (*src >= 'A' && *src <= 'Z') + else if (*src >= not_case_lower_bound && *src <= not_case_lower_bound) is_case = false; is_ascii = true; From ff7dc2635b6b3e46dbda5f1f4b7bbbf5de821c2d Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 29 May 2015 13:43:05 +0300 Subject: [PATCH 044/109] dbms: SSE lower/upper(UTF8) fix logical error, disable SSE4.2 [#METR-14764] --- dbms/include/DB/Functions/FunctionsString.h | 51 ++++++++++++--------- 1 file changed, 30 insertions(+), 21 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index 8f50c95e420..d976d3028d3 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -267,15 +267,16 @@ struct LowerUpperImplVectorized private: static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) { - const auto src_end_sse = src_end - (src_end - src) % 16; + const auto bytes_sse = sizeof(__m128i); + const auto src_end_sse = src_end - (src_end - src) % bytes_sse; - const auto flip_case_mask = 1 << 5; + const auto flip_case_mask = 'A' ^ 'a'; const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); const auto v_flip_case_mask = _mm_set1_epi8(flip_case_mask); - for (; src < src_end_sse; src += 16, dst += 16) + for (; src < src_end_sse; src += bytes_sse, dst += bytes_sse) { /// load 16 sequential 8-bit characters const auto chars = _mm_loadu_si128(reinterpret_cast(src)); @@ -295,7 +296,10 @@ private: } for (; src < src_end; ++src, ++dst) - *dst = (*src >= not_case_lower_bound && *src <= not_case_upper_bound) ? *src ^ flip_case_mask : *src; + if (*src >= not_case_lower_bound && *src <= not_case_upper_bound) + *dst = *src ^ flip_case_mask; + else + *dst = *src; } }; @@ -339,20 +343,23 @@ private: static bool isCaseASCII(const UInt8 * src, const UInt8 * const src_end, bool & is_ascii) { - const auto src_end_sse = src_end - (src_end - src) % 16; + const auto bytes_sse = sizeof(__m128i); + const auto src_end_sse = src_end - (src_end - src) % bytes_sse; -// const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); -// const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); - const auto v_not_case_range = _mm_set1_epi16((not_case_upper_bound << 8) | not_case_lower_bound); + const auto ascii_upper_bound = '\x7f'; + /// SSE2 packed comparison operate on signed types, hence compare (c < 0) instead of (c > 0x7f) const auto v_zero = _mm_setzero_si128(); + const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); + const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); +// const auto v_not_case_range = _mm_set1_epi16((not_case_upper_bound << 8) | not_case_lower_bound); auto is_case = true; - for (; src < src_end_sse; src += 16) + for (; src < src_end_sse; src += bytes_sse) { const auto chars = _mm_loadu_si128(reinterpret_cast(src)); - /// check for ASCII and case + /// check for ASCII const auto is_not_ascii = _mm_cmplt_epi8(chars, v_zero); const auto mask_is_not_ascii = _mm_movemask_epi8(is_not_ascii); @@ -362,26 +369,27 @@ private: return false; } - const auto is_case_result = _mm_cmpestra(v_not_case_range, 2, chars, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES); - if (is_case_result == 0) - is_case = false; - -// const auto is_not_case = _mm_and_si128(_mm_cmpgt_epi8(chars, v_not_case_lower_bound), -// _mm_cmplt_epi8(chars, v_not_case_upper_bound)); -// const auto mask_is_not_case = _mm_movemask_epi8(is_not_case); -// -// if (mask_is_not_case != 0) + /// check for case +// const auto is_case_result = _mm_cmpestra(v_not_case_range, 2, chars, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES); +// if (is_case_result == 0) // is_case = false; + + const auto is_not_case = _mm_and_si128(_mm_cmpgt_epi8(chars, v_not_case_lower_bound), + _mm_cmplt_epi8(chars, v_not_case_upper_bound)); + const auto mask_is_not_case = _mm_movemask_epi8(is_not_case); + + if (mask_is_not_case != 0) + is_case = false; } /// handle remaining symbols for (; src < src_end; ++src) - if (*src > '\x7f') + if (*src > ascii_upper_bound) { is_ascii = false; return false; } - else if (*src >= not_case_lower_bound && *src <= not_case_lower_bound) + else if (*src >= not_case_lower_bound && *src <= not_case_upper_bound) is_case = false; is_ascii = true; @@ -390,6 +398,7 @@ private: static void UTF8ToCase(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) { + /// @todo pessimistic variant, maybe SSE4.2 can help speeding it up a little bit (at least for cyrillic) static const Poco::UTF8Encoding utf8; while (src < src_end) From 78bda3cd50bfe58c3f68d0de9d1bf7a3cfc94722 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 29 May 2015 17:58:24 +0300 Subject: [PATCH 045/109] dbms: SSE lower/upperUTF8 finer granularity, more realistic approach [#METR-14764] --- dbms/include/DB/Functions/FunctionsString.h | 97 +++++++++++---------- 1 file changed, 49 insertions(+), 48 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index d976d3028d3..f9f0b6d27e9 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -331,31 +331,20 @@ struct LowerUpperUTF8ImplVectorized private: static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) { - auto is_ascii = false; + static const Poco::UTF8Encoding utf8; - if (isCaseASCII(src, src_end, is_ascii)) - std::copy(src, src_end, dst); - else if (is_ascii) - LowerUpperImplVectorized::array(src, src_end, dst); - else - UTF8ToCase(src, src_end, dst); - } - - static bool isCaseASCII(const UInt8 * src, const UInt8 * const src_end, bool & is_ascii) - { const auto bytes_sse = sizeof(__m128i); - const auto src_end_sse = src_end - (src_end - src) % bytes_sse; + auto src_end_sse = src + (src_end - src) * bytes_sse / bytes_sse; - const auto ascii_upper_bound = '\x7f'; + const auto flip_case_mask = 'A' ^ 'a'; /// SSE2 packed comparison operate on signed types, hence compare (c < 0) instead of (c > 0x7f) const auto v_zero = _mm_setzero_si128(); const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); // const auto v_not_case_range = _mm_set1_epi16((not_case_upper_bound << 8) | not_case_lower_bound); + const auto v_flip_case_mask = _mm_set1_epi8(flip_case_mask); - auto is_case = true; - - for (; src < src_end_sse; src += bytes_sse) + for (; src < src_end_sse; src += bytes_sse, dst += bytes_sse) { const auto chars = _mm_loadu_si128(reinterpret_cast(src)); @@ -363,44 +352,56 @@ private: const auto is_not_ascii = _mm_cmplt_epi8(chars, v_zero); const auto mask_is_not_ascii = _mm_movemask_epi8(is_not_ascii); - if (mask_is_not_ascii != 0) + /// ASCII + if (mask_is_not_ascii == 0) { - is_ascii = false; - return false; + const auto is_not_case = _mm_and_si128(_mm_cmpgt_epi8(chars, v_not_case_lower_bound), + _mm_cmplt_epi8(chars, v_not_case_upper_bound)); + const auto mask_is_not_case = _mm_movemask_epi8(is_not_case); + + /// check for case + // const auto is_case_result = _mm_cmpestra(v_not_case_range, 2, chars, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES); + // if (is_case_result == 0) + + /// not in case + if (mask_is_not_case != 0) + { + /// keep `flip_case_mask` only where necessary, zero out elsewhere + const auto xor_mask = _mm_and_si128(v_flip_case_mask, is_not_case); + + /// flip case by applying calculated mask + const auto cased_chars = _mm_xor_si128(chars, xor_mask); + + /// store result back to destination + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), cased_chars); + } + else + std::copy(src, src + bytes_sse, dst); } + else + { + /// UTF-8 + const auto end = src + bytes_sse; + while (src < end) + { + if (const auto chars = utf8.convert(to_case(utf8.convert(src)), dst, src_end_sse - src)) + { + src += chars; + dst += chars; + } + else + { + ++src; + ++dst; + } + } - /// check for case -// const auto is_case_result = _mm_cmpestra(v_not_case_range, 2, chars, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES); -// if (is_case_result == 0) -// is_case = false; - - const auto is_not_case = _mm_and_si128(_mm_cmpgt_epi8(chars, v_not_case_lower_bound), - _mm_cmplt_epi8(chars, v_not_case_upper_bound)); - const auto mask_is_not_case = _mm_movemask_epi8(is_not_case); - - if (mask_is_not_case != 0) - is_case = false; + const auto diff = src - end; + src_end_sse += diff; + } } /// handle remaining symbols - for (; src < src_end; ++src) - if (*src > ascii_upper_bound) - { - is_ascii = false; - return false; - } - else if (*src >= not_case_lower_bound && *src <= not_case_upper_bound) - is_case = false; - - is_ascii = true; - return is_case; - } - - static void UTF8ToCase(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) - { - /// @todo pessimistic variant, maybe SSE4.2 can help speeding it up a little bit (at least for cyrillic) - static const Poco::UTF8Encoding utf8; - while (src < src_end) { if (const auto chars = utf8.convert(to_case(utf8.convert(src)), dst, src_end - src)) From f16cdbdefa96141aa921534950ce57a1f6d198b2 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2015 19:09:51 +0300 Subject: [PATCH 046/109] dbms: tiny modifications for rounding functions [#METR-15210]. --- dbms/include/DB/Functions/FunctionsRound.h | 27 +++++----------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index ef9a1ed4d34..93c4478bae0 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -101,18 +101,14 @@ namespace DB * Без проверки, потому что делитель всегда положительный. */ template - struct FastModulo - { - }; + struct FastModulo; template struct FastModulo::value>::type> { private: template - struct Extend - { - }; + struct Extend; template struct Extend(a); U rem = val - (val / divisor.second) * static_cast(divisor.first); return static_cast(rem); @@ -172,9 +165,7 @@ namespace DB /** Реализация низкоуровневых функций округления для целочисленных значений. */ template - struct IntegerRoundingComputation - { - }; + struct IntegerRoundingComputation; template struct IntegerRoundingComputation - struct BaseFloatRoundingComputation - { - }; + struct BaseFloatRoundingComputation; template<> struct BaseFloatRoundingComputation @@ -494,9 +483,7 @@ namespace DB /** Реализация высокоуровневых функций округления. */ template - struct FunctionRoundingImpl - { - }; + struct FunctionRoundingImpl; /** Реализация высокоуровневых функций округления для целочисленных значений. */ @@ -683,9 +670,7 @@ namespace * умножения и деления. Поэтому оно называется масштабом. */ template - struct ScaleForRightType - { - }; + struct ScaleForRightType; template struct ScaleForRightType Date: Fri, 29 May 2015 19:46:46 +0300 Subject: [PATCH 047/109] dbms: fixed error [#METR-15210]. --- dbms/include/DB/Functions/FunctionsRound.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index 93c4478bae0..2aeb88cf908 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -606,7 +606,7 @@ namespace DB public: static inline void apply(const PODArray & in, size_t scale, typename ColumnVector::Container_t & out) { - ::memset(reinterpret_cast(&out[0]), 0, in.size()); + ::memset(reinterpret_cast(&out[0]), 0, in.size() * sizeof(T)); } static inline T apply(T val, size_t scale) From 3a6d28eb0eae05e68a0136f477f1d82c2630dc16 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2015 22:08:51 +0300 Subject: [PATCH 048/109] dbms: fixed error: empty queries in process list [#METR-16577]. --- dbms/src/Parsers/ParserSelectQuery.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/src/Parsers/ParserSelectQuery.cpp b/dbms/src/Parsers/ParserSelectQuery.cpp index 25d1746c2b7..0780028792f 100644 --- a/dbms/src/Parsers/ParserSelectQuery.cpp +++ b/dbms/src/Parsers/ParserSelectQuery.cpp @@ -16,7 +16,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Pos & max_p { Pos begin = pos; - ASTSelectQuery * select_query = new ASTSelectQuery(StringRange(begin, pos)); + ASTSelectQuery * select_query = new ASTSelectQuery; node = select_query; ParserWhiteSpaceOrComments ws; @@ -312,6 +312,8 @@ bool ParserSelectQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Pos & max_p ws.ignore(pos, end); } + select_query->range = StringRange(begin, pos); + select_query->children.push_back(select_query->select_expression_list); if (select_query->database) select_query->children.push_back(select_query->database); From 6a5ef9be832820b2482c257626f794a43f1f980a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 29 May 2015 23:36:09 +0300 Subject: [PATCH 049/109] dbms: fixed error with hanging INSERTs [#METR-16514]. --- dbms/src/Server/TCPHandler.cpp | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp index 1b784cd3505..85e4057c6bc 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/src/Server/TCPHandler.cpp @@ -225,14 +225,31 @@ void TCPHandler::readData(const Settings & global_settings) { while (1) { - /// Ждём пакета от клиента. При этом, каждые POLL_INTERVAL сек. проверяем, не требуется ли завершить работу. - while (!static_cast(*in).poll(global_settings.poll_interval * 1000000) && !Daemon::instance().isCancelled()) - ; + Stopwatch watch(CLOCK_MONOTONIC_COARSE); - /// Если требуется завершить работу, или клиент отсоединился. - if (Daemon::instance().isCancelled() || in->eof()) + /// Ждём пакета от клиента. При этом, каждые POLL_INTERVAL сек. проверяем, не требуется ли завершить работу. + while (1) + { + if (static_cast(*in).poll(global_settings.poll_interval * 1000000)) + break; + + /// Если требуется завершить работу. + if (Daemon::instance().isCancelled()) + return; + + /** Если ждём данных уже слишком долго. + * Если периодически poll-ить соединение, то receive_timeout у сокета сам по себе не срабатывает. + * Поэтому, добавлена дополнительная проверка. + */ + if (watch.elapsedSeconds() > global_settings.receive_timeout.totalSeconds()) + throw Exception("Timeout exceeded while receiving data from client", ErrorCodes::SOCKET_TIMEOUT); + } + + /// Если клиент отсоединился. + if (in->eof()) return; + /// Принимаем и обрабатываем данные. А если они закончились, то выходим. if (!receivePacket()) break; } From 96f1bbc379516089e207229840f4d3c6e6db60b6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 30 May 2015 00:32:10 +0300 Subject: [PATCH 050/109] dbms: added support for randomization of quota intervals [#METR-16576]. --- .../DataStreams/IProfilingBlockInputStream.h | 3 ++- dbms/include/DB/Interpreters/Quota.h | 7 +++--- .../IProfilingBlockInputStream.cpp | 1 + dbms/src/Interpreters/Quota.cpp | 22 +++++++++++++------ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/dbms/include/DB/DataStreams/IProfilingBlockInputStream.h b/dbms/include/DB/DataStreams/IProfilingBlockInputStream.h index 88374fb19e4..0b8fbe16a62 100644 --- a/dbms/include/DB/DataStreams/IProfilingBlockInputStream.h +++ b/dbms/include/DB/DataStreams/IProfilingBlockInputStream.h @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -14,6 +13,8 @@ namespace DB { +class QuotaForIntervals; + /** Смотрит за тем, как работает источник блоков. * Позволяет получить информацию для профайлинга: diff --git a/dbms/include/DB/Interpreters/Quota.h b/dbms/include/DB/Interpreters/Quota.h index 6f3de23f4d4..4c03ecd41a6 100644 --- a/dbms/include/DB/Interpreters/Quota.h +++ b/dbms/include/DB/Interpreters/Quota.h @@ -72,13 +72,14 @@ struct QuotaForInterval { time_t rounded_time; size_t duration; + time_t offset; /// Смещение интервала, для рандомизации. QuotaValues max; QuotaValues used; QuotaForInterval() : rounded_time() {} QuotaForInterval(time_t duration_) : duration(duration_) {} - void initFromConfig(const String & config_elem, time_t duration_, Poco::Util::AbstractConfiguration & config); + void initFromConfig(const String & config_elem, time_t duration_, time_t offset_, Poco::Util::AbstractConfiguration & config); /// Увеличить соответствующее значение. void addQuery(time_t current_time, const String & quota_name); @@ -131,7 +132,7 @@ public: return cont.empty(); } - void initFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config); + void initFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config, std::mt19937 & rng); /// Обновляет максимальные значения значениями из quota. /// Удаляет интервалы, которых нет в quota, добавляет интревалы, которых нет здесь, но есть в quota. @@ -177,7 +178,7 @@ struct Quota Quota() : is_keyed(false), keyed_by_ip(false) {} - void loadFromConfig(const String & config_elem, const String & name_, Poco::Util::AbstractConfiguration & config); + void loadFromConfig(const String & config_elem, const String & name_, Poco::Util::AbstractConfiguration & config, std::mt19937 & rng); QuotaForIntervalsPtr get(const String & quota_key, const String & user_name, const Poco::Net::IPAddress & ip); }; diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp index 97f831cd582..c8ae1a3e999 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp @@ -4,6 +4,7 @@ #include */ #include +#include #include diff --git a/dbms/src/Interpreters/Quota.cpp b/dbms/src/Interpreters/Quota.cpp index 44ef33d4eca..af59ddd2afa 100644 --- a/dbms/src/Interpreters/Quota.cpp +++ b/dbms/src/Interpreters/Quota.cpp @@ -24,10 +24,11 @@ void QuotaValues::initFromConfig(const String & config_elem, Poco::Util::Abstrac } -void QuotaForInterval::initFromConfig(const String & config_elem, time_t duration_, Poco::Util::AbstractConfiguration & config) +void QuotaForInterval::initFromConfig(const String & config_elem, time_t duration_, time_t offset_, Poco::Util::AbstractConfiguration & config) { rounded_time = 0; duration = duration_; + offset = offset_; max.initFromConfig(config_elem, config); } @@ -95,7 +96,7 @@ void QuotaForInterval::updateTime(time_t current_time) { if (current_time >= rounded_time + static_cast(duration)) { - rounded_time = current_time / duration * duration; + rounded_time = (current_time - offset) / duration * duration + offset; used.clear(); } } @@ -127,7 +128,7 @@ void QuotaForInterval::check(size_t max_amount, size_t used_amount, time_t curre } -void QuotaForIntervals::initFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config) +void QuotaForIntervals::initFromConfig(const String & config_elem, Poco::Util::AbstractConfiguration & config, std::mt19937 & rng) { Poco::Util::AbstractConfiguration::Keys config_keys; config.keys(config_elem, config_keys); @@ -139,8 +140,13 @@ void QuotaForIntervals::initFromConfig(const String & config_elem, Poco::Util::A String interval_config_elem = config_elem + "." + *it; time_t duration = config.getInt(interval_config_elem + ".duration"); + time_t offset = 0; - cont[duration].initFromConfig(interval_config_elem, duration, config); + bool randomize = config.getBool(interval_config_elem + ".randomize", false); + if (randomize) + offset = std::uniform_int_distribution(0, duration - 1)(rng); + + cont[duration].initFromConfig(interval_config_elem, duration, offset, config); } } @@ -210,7 +216,7 @@ String QuotaForIntervals::toString() const } -void Quota::loadFromConfig(const String & config_elem, const String & name_, Poco::Util::AbstractConfiguration & config) +void Quota::loadFromConfig(const String & config_elem, const String & name_, Poco::Util::AbstractConfiguration & config, std::mt19937 & rng) { name = name_; @@ -226,7 +232,7 @@ void Quota::loadFromConfig(const String & config_elem, const String & name_, Poc } QuotaForIntervals new_max(name); - new_max.initFromConfig(config_elem, config); + new_max.initFromConfig(config_elem, config, rng); if (!(new_max == max)) { max = new_max; @@ -269,6 +275,8 @@ QuotaForIntervalsPtr Quota::get(const String & quota_key, const String & user_na void Quotas::loadFromConfig(Poco::Util::AbstractConfiguration & config) { + std::mt19937 rng; + Poco::Util::AbstractConfiguration::Keys config_keys; config.keys("quotas", config_keys); @@ -286,7 +294,7 @@ void Quotas::loadFromConfig(Poco::Util::AbstractConfiguration & config) { if (!cont[*it]) cont[*it] = new Quota(); - cont[*it]->loadFromConfig("quotas." + *it, *it, config); + cont[*it]->loadFromConfig("quotas." + *it, *it, config, rng); } } From 68cfd47195f9d4102c4fc78d8908d13806b3c643 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 30 May 2015 00:33:18 +0300 Subject: [PATCH 051/109] dbms: added missing aggregate function synonim [#METR-16266]. --- dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index 61b81fc37a5..40db1f5f693 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -468,7 +468,7 @@ AggregateFunctionPtr AggregateFunctionFactory::get(const String & name, const Da return res; } - else if (name == "quantileDeterministic") + else if (name == "medianDeterministic" || name == "quantileDeterministic") { if (argument_types.size() != 2) throw Exception("Incorrect number of arguments for aggregate function " + name, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); @@ -722,6 +722,7 @@ const AggregateFunctionFactory::FunctionNames & AggregateFunctionFactory::getFun "quantileTimingWeighted", "quantilesTimingWeighted", "medianTimingWeighted", + "medianDeterministic", "quantileDeterministic", "quantilesDeterministic", "sequenceMatch", From 9c2cf55a28de4f7b6990ad4f2e9f698e29e00701 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 30 May 2015 00:37:17 +0300 Subject: [PATCH 052/109] dbms: addition to prev. revision [#METR-16576]. --- dbms/src/Server/HTTPHandler.cpp | 1 + dbms/src/Server/TCPHandler.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/dbms/src/Server/HTTPHandler.cpp b/dbms/src/Server/HTTPHandler.cpp index 04a15723b64..cd13d00c57c 100644 --- a/dbms/src/Server/HTTPHandler.cpp +++ b/dbms/src/Server/HTTPHandler.cpp @@ -17,6 +17,7 @@ #include #include +#include #include diff --git a/dbms/src/Server/TCPHandler.cpp b/dbms/src/Server/TCPHandler.cpp index 85e4057c6bc..233918f14a5 100644 --- a/dbms/src/Server/TCPHandler.cpp +++ b/dbms/src/Server/TCPHandler.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include From 018a31f416bf3638d67984355c8e2594340855ea Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Mon, 1 Jun 2015 15:50:47 +0300 Subject: [PATCH 053/109] dbms: SSE lower/upperUTF8 fix calculation of array end aligned to a multiple of 16 [#METR-14764] --- dbms/include/DB/Functions/FunctionsString.h | 86 ++++++++++++++++----- 1 file changed, 65 insertions(+), 21 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index f9f0b6d27e9..a3c530217a9 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -334,9 +334,10 @@ private: static const Poco::UTF8Encoding utf8; const auto bytes_sse = sizeof(__m128i); - auto src_end_sse = src + (src_end - src) * bytes_sse / bytes_sse; + auto src_end_sse = src + (src_end - src) / bytes_sse * bytes_sse; const auto flip_case_mask = 'A' ^ 'a'; + const auto ascii_upper_bound = '\x7f'; /// SSE2 packed comparison operate on signed types, hence compare (c < 0) instead of (c > 0x7f) const auto v_zero = _mm_setzero_si128(); const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); @@ -344,7 +345,7 @@ private: // const auto v_not_case_range = _mm_set1_epi16((not_case_upper_bound << 8) | not_case_lower_bound); const auto v_flip_case_mask = _mm_set1_epi8(flip_case_mask); - for (; src < src_end_sse; src += bytes_sse, dst += bytes_sse) + while (src < src_end_sse) { const auto chars = _mm_loadu_si128(reinterpret_cast(src)); @@ -363,9 +364,12 @@ private: // const auto is_case_result = _mm_cmpestra(v_not_case_range, 2, chars, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES); // if (is_case_result == 0) - /// not in case - if (mask_is_not_case != 0) + /// everything in correct case ASCII + if (mask_is_not_case == 0) + _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), chars); + else { + /// ASCII in mixed case /// keep `flip_case_mask` only where necessary, zero out elsewhere const auto xor_mask = _mm_and_si128(v_flip_case_mask, is_not_case); @@ -375,24 +379,72 @@ private: /// store result back to destination _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), cased_chars); } - else - std::copy(src, src + bytes_sse, dst); + + src += bytes_sse, dst += bytes_sse; } else { /// UTF-8 const auto end = src + bytes_sse; + while (src < end) { - if (const auto chars = utf8.convert(to_case(utf8.convert(src)), dst, src_end_sse - src)) + if (src[0] <= ascii_upper_bound) { - src += chars; - dst += chars; + if (*src >= not_case_lower_bound && *src <= not_case_upper_bound) + *dst++ = *src++ ^ flip_case_mask; + else + *dst++ = *src++; } +// else if (src + 1 < src_end /// кириллица: русский алфавит, а также ЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏ +// && ((src[0] == '\xD0' && (src[1] >= '\x80' && src[1] <= '\xBF')) +// || (src[0] == '\xD1' && (src[1] >= '\x80' && src[1] <= '\x9F')))) +// { +// /** d0 80-8f -> +1 +10 +// * d0 90-9f -> +0 +20 +// * d0 a0-af -> +1 -20 +// */ +// +// if (src[0] == '\xD0' && (src[1] >= '\x80' && src[1] <= '\x8F')) +// { +// ++res_pos[0]; +// res_pos[1] += 0x10; +// } +// else if (src[0] == '\xD0' && (src[1] >= '\x90' && src[1] <= '\x9F')) +// { +// res_pos[1] += 0x20; +// } +// else if (src[0] == '\xD0' && (src[1] >= '\xA0' && src[1] <= '\xAF')) +// { +// ++res_pos[0]; +// res_pos[1] -= 0x20; +// } +// +// pos += 2; +// res_pos += 2; +// } +// else if (pos + 1 < src_end && pos[0] == '\xC2') /// Пунктуация U+0080 - U+00BF, UTF-8: C2 80 - C2 BF +// { +// pos += 2; +// res_pos += 2; +// } +// else if (pos + 2 < src_end && pos[0] == '\xE2') /// Символы U+2000 - U+2FFF, UTF-8: E2 80 80 - E2 BF BF +// { +// pos += 3; +// res_pos += 3; +// } else { - ++src; - ++dst; + if (const auto chars = utf8.convert(to_case(utf8.convert(src)), dst, src_end - src)) + { + src += chars; + dst += chars; + } + else + { + ++src; + ++dst; + } } } @@ -403,18 +455,10 @@ private: /// handle remaining symbols while (src < src_end) - { if (const auto chars = utf8.convert(to_case(utf8.convert(src)), dst, src_end - src)) - { - src += chars; - dst += chars; - } + src += chars, dst += chars; else - { - ++src; - ++dst; - } - } + ++src, ++dst; } }; From 20fe37511f8a5b8ceb7f89b22dede1d01c074098 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Tue, 2 Jun 2015 14:16:02 +0300 Subject: [PATCH 054/109] Merge --- dbms/include/DB/Parsers/ASTSelectQuery.h | 171 +++-------------- dbms/src/Parsers/ASTSelectQuery.cpp | 179 ++++++++++++++++++ .../0_stateless/00098_j_union_all.reference | 4 + .../queries/0_stateless/00098_j_union_all.sql | 2 + 4 files changed, 209 insertions(+), 147 deletions(-) create mode 100644 dbms/src/Parsers/ASTSelectQuery.cpp create mode 100644 dbms/tests/queries/0_stateless/00098_j_union_all.reference create mode 100644 dbms/tests/queries/0_stateless/00098_j_union_all.sql diff --git a/dbms/include/DB/Parsers/ASTSelectQuery.h b/dbms/include/DB/Parsers/ASTSelectQuery.h index 20b2065310a..fd7d6d71d48 100644 --- a/dbms/include/DB/Parsers/ASTSelectQuery.h +++ b/dbms/include/DB/Parsers/ASTSelectQuery.h @@ -9,11 +9,34 @@ namespace DB { - /** SELECT запрос */ class ASTSelectQuery : public ASTQueryWithOutput { +public: + ASTSelectQuery() = default; + ASTSelectQuery(const StringRange range_); + + /** Получить текст, который идентифицирует этот элемент. */ + String getID() const override { return "SelectQuery"; }; + + /// Проверить наличие функции arrayJoin. (Не большого ARRAY JOIN.) + static bool hasArrayJoin(const ASTPtr & ast); + + /// Содержит ли запрос астериск? + bool hasAsterisk() const; + + /// Переименовать столбцы запроса в такие же имена, как в исходном запросе. + void renameColumns(const ASTSelectQuery & source); + + /// Переписывает select_expression_list, чтобы вернуть только необходимые столбцы в правильном порядке. + void rewriteSelectExpressionList(const Names & column_names); + + /// Переписывает select_expression_list, чтобы вернуть только необходимые столбцы в правильном порядке. + void rewriteSelectExpressionList3(const Names & column_names); + + ASTPtr clone() const override; + public: bool distinct = false; ASTPtr select_expression_list; @@ -32,152 +55,6 @@ public: ASTPtr limit_offset; ASTPtr limit_length; ASTPtr next_union_all; /// Следующий запрос SELECT в цепочке UNION ALL, если такой есть - - ASTSelectQuery() = default; - ASTSelectQuery(const StringRange range_) : ASTQueryWithOutput(range_) {} - - /** Получить текст, который идентифицирует этот элемент. */ - String getID() const override { return "SelectQuery"; }; - - /// Проверить наличие функции arrayJoin. (Не большого ARRAY JOIN.) - static bool hasArrayJoin(const ASTPtr & ast) - { - if (const ASTFunction * function = typeid_cast(&*ast)) - if (function->kind == ASTFunction::ARRAY_JOIN) - return true; - - for (const auto & child : ast->children) - if (hasArrayJoin(child)) - return true; - - return false; - } - - /// Содержит ли запрос астериск? - bool hasAsterisk() const - { - for (const auto & ast : select_expression_list->children) - if (typeid_cast(&*ast) != nullptr) - return true; - - return false; - } - - /// Переименовать столбцы запроса в такие же имена, как в исходном запросе. - void renameColumns(const ASTSelectQuery & source) - { - const ASTs & from = source.select_expression_list->children; - ASTs & to = select_expression_list->children; - - if (from.size() != to.size()) - throw Exception("Size mismatch in UNION ALL chain", - DB::ErrorCodes::UNION_ALL_RESULT_STRUCTURES_MISMATCH); - - for (size_t i = 0; i < from.size(); ++i) - { - /// Если столбец имеет алиас, то он должен совпадать с названием исходного столбца. - /// В противном случае мы ему присваиваем алиас, если требуется. - if (!to[i]->tryGetAlias().empty()) - { - if (to[i]->tryGetAlias() != from[i]->getAliasOrColumnName()) - throw Exception("Column alias mismatch in UNION ALL chain", - DB::ErrorCodes::UNION_ALL_COLUMN_ALIAS_MISMATCH); - } - else if (to[i]->getColumnName() != from[i]->getAliasOrColumnName()) - to[i]->setAlias(from[i]->getAliasOrColumnName()); - } - } - - /// Переписывает select_expression_list, чтобы вернуть только необходимые столбцы в правильном порядке. - void rewriteSelectExpressionList(const Names & column_names) - { - ASTPtr result = new ASTExpressionList; - ASTs asts = select_expression_list->children; - - /// Не будем выбрасывать выражения, содержащие функцию arrayJoin. - std::set unremovable_asts; - for (size_t j = 0; j < asts.size(); ++j) - { - if (hasArrayJoin(asts[j])) - { - result->children.push_back(asts[j]->clone()); - unremovable_asts.insert(asts[j]); - } - } - - for (const auto & column_name : column_names) - { - bool done = false; - for (size_t j = 0; j < asts.size(); ++j) - { - if (asts[j]->getAliasOrColumnName() == column_name) - { - if (!unremovable_asts.count(asts[j])) - result->children.push_back(asts[j]->clone()); - done = true; - } - } - if (!done) - throw Exception("Error while rewriting expression list for select query." - " Could not find alias: " + column_name, - DB::ErrorCodes::UNKNOWN_IDENTIFIER); - } - - for (auto & child : children) - { - if (child == select_expression_list) - { - child = result; - break; - } - } - select_expression_list = result; - - /** NOTE: Может показаться, что мы могли испортить запрос, выбросив выражение с алиасом, который используется где-то еще. - * Такого произойти не может, потому что этот метод вызывается всегда для запроса, на котором хоть раз создавали - * ExpressionAnalyzer, что гарантирует, что в нем все алиасы уже подставлены. Не совсем очевидная логика :) - */ - } - - ASTPtr clone() const override - { - ASTSelectQuery * res = new ASTSelectQuery(*this); - ASTPtr ptr{res}; - - res->children.clear(); - -#define CLONE(member) if (member) { res->member = member->clone(); res->children.push_back(res->member); } - - /** NOTE Члены должны клонироваться точно в таком же порядке, - * в каком они были вставлены в children в ParserSelectQuery. - * Это важно, потому что из имён children-ов составляется идентификатор (getTreeID), - * который может быть использован для идентификаторов столбцов в случае подзапросов в операторе IN. - * При распределённой обработке запроса, в случае, если один из серверов localhost, а другой - нет, - * запрос на localhost выполняется в рамках процесса и при этом клонируется, - * а на удалённый сервер запрос отправляется в текстовом виде по TCP. - * И если порядок при клонировании не совпадает с порядком при парсинге, - * то на разных серверах получатся разные идентификаторы. - */ - CLONE(select_expression_list) - CLONE(database) - CLONE(table) - CLONE(array_join_expression_list) - CLONE(join) - CLONE(sample_size) - CLONE(prewhere_expression) - CLONE(where_expression) - CLONE(group_expression_list) - CLONE(having_expression) - CLONE(order_expression_list) - CLONE(limit_offset) - CLONE(limit_length) - CLONE(format) - CLONE(next_union_all) - -#undef CLONE - - return ptr; - } }; } diff --git a/dbms/src/Parsers/ASTSelectQuery.cpp b/dbms/src/Parsers/ASTSelectQuery.cpp new file mode 100644 index 00000000000..aec0d71df84 --- /dev/null +++ b/dbms/src/Parsers/ASTSelectQuery.cpp @@ -0,0 +1,179 @@ +#include + +namespace DB +{ + +ASTSelectQuery::ASTSelectQuery(const StringRange range_) : ASTQueryWithOutput(range_) +{ +} + +bool ASTSelectQuery::hasArrayJoin(const ASTPtr & ast) +{ + if (const ASTFunction * function = typeid_cast(&*ast)) + if (function->kind == ASTFunction::ARRAY_JOIN) + return true; + + for (const auto & child : ast->children) + if (hasArrayJoin(child)) + return true; + + return false; +} + +bool ASTSelectQuery::hasAsterisk() const +{ + for (const auto & ast : select_expression_list->children) + if (typeid_cast(&*ast) != nullptr) + return true; + + return false; +} + +void ASTSelectQuery::renameColumns(const ASTSelectQuery & source) +{ + const ASTs & from = source.select_expression_list->children; + ASTs & to = select_expression_list->children; + + if (from.size() != to.size()) + throw Exception("Size mismatch in UNION ALL chain", + DB::ErrorCodes::UNION_ALL_RESULT_STRUCTURES_MISMATCH); + + for (size_t i = 0; i < from.size(); ++i) + { + /// Если столбец имеет алиас, то он должен совпадать с названием исходного столбца. + /// В противном случае мы ему присваиваем алиас, если требуется. + if (!to[i]->tryGetAlias().empty()) + { + if (to[i]->tryGetAlias() != from[i]->getAliasOrColumnName()) + throw Exception("Column alias mismatch in UNION ALL chain", + DB::ErrorCodes::UNION_ALL_COLUMN_ALIAS_MISMATCH); + } + else if (to[i]->getColumnName() != from[i]->getAliasOrColumnName()) + to[i]->setAlias(from[i]->getAliasOrColumnName()); + } +} + +void ASTSelectQuery::rewriteSelectExpressionList(const Names & column_names) +{ + ASTPtr result = new ASTExpressionList; + ASTs asts = select_expression_list->children; + + /// Создать отображение. + + /// Элемент отображения. + struct Arrow + { + Arrow() = default; + Arrow(size_t to_position_) : + to_position(to_position_), is_selected(true) + { + } + size_t to_position = 0; + bool is_selected = false; + }; + + /// Отображение одного SELECT выражения в другое. + using Mapping = std::vector; + + Mapping mapping(asts.size()); + std::vector from(column_names.size()); + + /// Не будем выбрасывать выражения, содержащие функцию arrayJoin. + for (size_t i = 0; i < asts.size(); ++i) + { + if (hasArrayJoin(asts[i])) + mapping[i] = Arrow(i); + } + + for (size_t i = 0; i < column_names.size(); ++i) + { + bool done = false; + for (size_t j = 0; j < asts.size(); ++j) + { + if (asts[j]->getAliasOrColumnName() == column_names[i]) + { + from[i] = j; + done = true; + break; + } + } + if (!done) + throw Exception("Error while rewriting expression list for select query." + " Could not find alias: " + column_names[i], + DB::ErrorCodes::UNKNOWN_IDENTIFIER); + } + + auto to = from; + std::sort(from.begin(), from.end()); + + for (size_t i = 0; i < column_names.size(); ++i) + { + auto & arrow = mapping[from[i]]; + if (!arrow.is_selected) + arrow = Arrow(to[i]); + } + + /// Составить новое выражение. + for (const auto & arrow : mapping) + { + if (arrow.is_selected) + result->children.push_back(asts[arrow.to_position]->clone()); + } + + for (auto & child : children) + { + if (child == select_expression_list) + { + child = result; + break; + } + } + select_expression_list = result; + + /** NOTE: Может показаться, что мы могли испортить запрос, выбросив выражение с алиасом, который используется где-то еще. + * Такого произойти не может, потому что этот метод вызывается всегда для запроса, на котором хоть раз создавали + * ExpressionAnalyzer, что гарантирует, что в нем все алиасы уже подставлены. Не совсем очевидная логика :) + */ +} + +ASTPtr ASTSelectQuery::clone() const +{ + ASTSelectQuery * res = new ASTSelectQuery(*this); + ASTPtr ptr{res}; + + res->children.clear(); + +#define CLONE(member) if (member) { res->member = member->clone(); res->children.push_back(res->member); } + + /** NOTE Члены должны клонироваться точно в таком же порядке, + * в каком они были вставлены в children в ParserSelectQuery. + * Это важно, потому что из имён children-ов составляется идентификатор (getTreeID), + * который может быть использован для идентификаторов столбцов в случае подзапросов в операторе IN. + * При распределённой обработке запроса, в случае, если один из серверов localhost, а другой - нет, + * запрос на localhost выполняется в рамках процесса и при этом клонируется, + * а на удалённый сервер запрос отправляется в текстовом виде по TCP. + * И если порядок при клонировании не совпадает с порядком при парсинге, + * то на разных серверах получатся разные идентификаторы. + */ + CLONE(select_expression_list) + CLONE(database) + CLONE(table) + CLONE(array_join_expression_list) + CLONE(join) + CLONE(sample_size) + CLONE(prewhere_expression) + CLONE(where_expression) + CLONE(group_expression_list) + CLONE(having_expression) + CLONE(order_expression_list) + CLONE(limit_offset) + CLONE(limit_length) + CLONE(format) + CLONE(next_union_all) + +#undef CLONE + + return ptr; +} +}; + diff --git a/dbms/tests/queries/0_stateless/00098_j_union_all.reference b/dbms/tests/queries/0_stateless/00098_j_union_all.reference new file mode 100644 index 00000000000..bf2cc015aaa --- /dev/null +++ b/dbms/tests/queries/0_stateless/00098_j_union_all.reference @@ -0,0 +1,4 @@ +0 -1 +0 -1 +-1 0 +-1 0 diff --git a/dbms/tests/queries/0_stateless/00098_j_union_all.sql b/dbms/tests/queries/0_stateless/00098_j_union_all.sql new file mode 100644 index 00000000000..2e450c0a53a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00098_j_union_all.sql @@ -0,0 +1,2 @@ +SELECT * FROM (SELECT dummy, -1 as x UNION ALL SELECT dummy, arrayJoin([-1]) as x); +SELECT * FROM (SELECT -1 as x, dummy UNION ALL SELECT arrayJoin([-1]) as x, dummy); From 82a330f47a4f6d714ac80c445c9ceef418d8eb34 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Tue, 2 Jun 2015 17:18:14 +0300 Subject: [PATCH 055/109] dbms: Server: Small simplification. [#METR-16546] --- dbms/src/Parsers/ASTSelectQuery.cpp | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/dbms/src/Parsers/ASTSelectQuery.cpp b/dbms/src/Parsers/ASTSelectQuery.cpp index aec0d71df84..3bbeb504a4e 100644 --- a/dbms/src/Parsers/ASTSelectQuery.cpp +++ b/dbms/src/Parsers/ASTSelectQuery.cpp @@ -107,11 +107,7 @@ void ASTSelectQuery::rewriteSelectExpressionList(const Names & column_names) std::sort(from.begin(), from.end()); for (size_t i = 0; i < column_names.size(); ++i) - { - auto & arrow = mapping[from[i]]; - if (!arrow.is_selected) - arrow = Arrow(to[i]); - } + mapping[from[i]] = Arrow(to[i]); /// Составить новое выражение. for (const auto & arrow : mapping) From 5752d8f561786d933e26c54d2c80ba189d5cec0e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Jun 2015 23:21:03 +0300 Subject: [PATCH 056/109] dbms: removing obsolete parts [#METR-16629]. --- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index 57946e82f64..b07d13279b8 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -770,6 +770,7 @@ MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( if (obsolete) { LOG_WARNING(log, "Obsolete part " + part->name + " added"); + part->remove_time = time(0); } else { From c19193cf611e9847c2741bd504fc6e1e652911d5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 2 Jun 2015 23:22:53 +0300 Subject: [PATCH 057/109] Merge --- .../DB/Storages/MergeTree/MergeTreeData.h | 9 +++++---- dbms/include/DB/Storages/StorageMergeTree.h | 5 ++++- dbms/src/Storages/MergeTree/MergeTreeData.cpp | 9 ++++++--- dbms/src/Storages/StorageMergeTree.cpp | 18 +++++------------- 4 files changed, 20 insertions(+), 21 deletions(-) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h index 73cf5907a7f..998c8d6f8fd 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeData.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeData.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include #include @@ -20,6 +18,9 @@ #define MERGE_TREE_MARK_SIZE (2 * sizeof(size_t)) +struct SimpleIncrement; + + namespace DB { @@ -743,12 +744,12 @@ public: * Предполагается, что кусок не пересекается с существующими. * Если out_transaction не nullptr, присваивает туда объект, позволяющий откатить добавление куска (но не переименование). */ - void renameTempPartAndAdd(MutableDataPartPtr & part, Increment * increment = nullptr, Transaction * out_transaction = nullptr); + void renameTempPartAndAdd(MutableDataPartPtr & part, SimpleIncrement * increment = nullptr, Transaction * out_transaction = nullptr); /** То же, что renameTempPartAndAdd, но кусок может покрывать существующие куски. * Удаляет и возвращает все куски, покрытые добавляемым (в возрастающем порядке). */ - DataPartsVector renameTempPartAndReplace(MutableDataPartPtr & part, Increment * increment = nullptr, Transaction * out_transaction = nullptr); + DataPartsVector renameTempPartAndReplace(MutableDataPartPtr & part, SimpleIncrement * increment = nullptr, Transaction * out_transaction = nullptr); /** Убирает из рабочего набора куски remove и добавляет куски add. add должны уже быть в all_data_parts. * Если clear_without_timeout, данные будут удалены при следующем clearOldParts, игнорируя old_parts_lifetime. diff --git a/dbms/include/DB/Storages/StorageMergeTree.h b/dbms/include/DB/Storages/StorageMergeTree.h index d5d657f3c3a..ec86c004d97 100644 --- a/dbms/include/DB/Storages/StorageMergeTree.h +++ b/dbms/include/DB/Storages/StorageMergeTree.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB @@ -107,7 +108,6 @@ private: String database_name; String table_name; String full_path; - Increment increment; Context & context; BackgroundProcessingPool & background_pool; @@ -117,6 +117,9 @@ private: MergeTreeDataWriter writer; MergeTreeDataMerger merger; + /// Для нумерации блоков. + SimpleIncrement increment; + MergeTreeData::DataParts currently_merging; Poco::FastMutex currently_merging_mutex; diff --git a/dbms/src/Storages/MergeTree/MergeTreeData.cpp b/dbms/src/Storages/MergeTree/MergeTreeData.cpp index b07d13279b8..e1b4cf42a09 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeData.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeData.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -681,7 +682,7 @@ MergeTreeData::AlterDataPartTransaction::~AlterDataPartTransaction() } -void MergeTreeData::renameTempPartAndAdd(MutableDataPartPtr & part, Increment * increment, Transaction * out_transaction) +void MergeTreeData::renameTempPartAndAdd(MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction) { auto removed = renameTempPartAndReplace(part, increment, out_transaction); if (!removed.empty()) @@ -692,7 +693,7 @@ void MergeTreeData::renameTempPartAndAdd(MutableDataPartPtr & part, Increment * } MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( - MutableDataPartPtr & part, Increment * increment, Transaction * out_transaction) + MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction) { if (out_transaction && out_transaction->data) throw Exception("Using the same MergeTreeData::Transaction for overlapping transactions is invalid", ErrorCodes::LOGICAL_ERROR); @@ -710,7 +711,7 @@ MergeTreeData::DataPartsVector MergeTreeData::renameTempPartAndReplace( * содержат ещё не добавленный кусок. */ if (increment) - part->left = part->right = increment->get(false); + part->left = part->right = increment->get(); String new_name = ActiveDataPartSet::getPartName(part->left_date, part->right_date, part->left, part->right, part->level); @@ -814,7 +815,9 @@ void MergeTreeData::attachPart(const DataPartPtr & part) if (!all_data_parts.insert(part).second) throw Exception("Part " + part->name + " is already attached", ErrorCodes::DUPLICATE_DATA_PART); + data_parts.insert(part); + addPartContributionToColumnSizes(part); } void MergeTreeData::renameAndDetachPart(const DataPartPtr & part, const String & prefix, bool restore_covered, bool move_to_detached) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 6cdbd13b56d..397a535fe67 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -29,18 +29,17 @@ StorageMergeTree::StorageMergeTree( const MergeTreeSettings & settings_) : IStorage{materialized_columns_, alias_columns_, column_defaults_}, path(path_), database_name(database_name_), table_name(table_name_), full_path(path + escapeForFileName(table_name) + '/'), - increment(full_path + "increment.txt"), context(context_), background_pool(context_.getBackgroundPool()), + context(context_), background_pool(context_.getBackgroundPool()), data(full_path, columns_, materialized_columns_, alias_columns_, column_defaults_, context_, primary_expr_ast_, date_column_name_, sampling_expression_, index_granularity_,mode_, sign_column_, columns_to_sum_, settings_, database_name_ + "." + table_name, false), reader(data), writer(data), merger(data), + increment(data.getMaxDataPartIndex()), log(&Logger::get(database_name_ + "." + table_name + " (StorageMergeTree)")), shutdown_called(false) { - increment.fixIfBroken(data.getMaxDataPartIndex()); - data.loadDataParts(false); data.clearOldParts(); } @@ -130,8 +129,6 @@ void StorageMergeTree::rename(const String & new_path_to_db, const String & new_ table_name = new_table_name; full_path = new_full_path; - increment.setPath(full_path + "increment.txt"); - /// TODO: Можно обновить названия логгеров у this, data, reader, writer, merger. } @@ -341,16 +338,11 @@ void StorageMergeTree::attachPartition(const Field & field, bool unreplicated, b LOG_DEBUG(log, "Checking data"); MergeTreeData::MutableDataPartPtr part = data.loadPartAndFixMetadata(source_path); - UInt64 index = increment.get(); - String new_part_name = ActiveDataPartSet::getPartName(part->left_date, part->right_date, index, index, 0); - part->renameTo(new_part_name); - part->name = new_part_name; - ActiveDataPartSet::parsePartName(part->name, *part); - - LOG_INFO(log, "Attaching part " << source_part_name << " from " << source_path << " as " << new_part_name); + LOG_INFO(log, "Attaching part " << source_part_name << " from " << source_path); + data.renameTempPartAndAdd(part, &increment); data.attachPart(part); - LOG_INFO(log, "Finished attaching part " << new_part_name); + LOG_INFO(log, "Finished attaching part"); } /// На месте удаленных кусков могут появиться новые, с другими данными. From 76bfd2af214bb75ae8ec3a385b5e03a84d94b221 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Jun 2015 00:08:12 +0300 Subject: [PATCH 058/109] dbms: addition to prev. revision [#METR-16629]. --- dbms/src/Storages/StorageMergeTree.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 397a535fe67..994dc8b8cd9 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -36,12 +36,13 @@ StorageMergeTree::StorageMergeTree( sampling_expression_, index_granularity_,mode_, sign_column_, columns_to_sum_, settings_, database_name_ + "." + table_name, false), reader(data), writer(data), merger(data), - increment(data.getMaxDataPartIndex()), + increment(0), log(&Logger::get(database_name_ + "." + table_name + " (StorageMergeTree)")), shutdown_called(false) { data.loadDataParts(false); data.clearOldParts(); + increment.set(data.getMaxDataPartIndex()); } StoragePtr StorageMergeTree::create( From 9c78e95702acdfadbe37bd36ba0f326925eedaef Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Jun 2015 00:24:41 +0300 Subject: [PATCH 059/109] dbms: addition to prev. revision [#METR-16629]. --- dbms/src/Storages/StorageMergeTree.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 994dc8b8cd9..66f55601524 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -341,7 +341,6 @@ void StorageMergeTree::attachPartition(const Field & field, bool unreplicated, b LOG_INFO(log, "Attaching part " << source_part_name << " from " << source_path); data.renameTempPartAndAdd(part, &increment); - data.attachPart(part); LOG_INFO(log, "Finished attaching part"); } From c341bd2c609e77ca2ca9e4289d27047d16c51780 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Jun 2015 06:29:28 +0300 Subject: [PATCH 060/109] dbms: function 'transform': added support for non-const argument with default value [#METR-15987]. --- .../DB/Functions/FunctionsMiscellaneous.h | 505 ----------- .../include/DB/Functions/FunctionsTransform.h | 804 ++++++++++++++++++ .../DataStreams/VerticalRowOutputStream.cpp | 4 +- dbms/src/Functions/FunctionsMiscellaneous.cpp | 67 -- dbms/src/Functions/FunctionsTransform.cpp | 84 ++ 5 files changed, 890 insertions(+), 574 deletions(-) create mode 100644 dbms/include/DB/Functions/FunctionsTransform.h create mode 100644 dbms/src/Functions/FunctionsTransform.cpp diff --git a/dbms/include/DB/Functions/FunctionsMiscellaneous.h b/dbms/include/DB/Functions/FunctionsMiscellaneous.h index f3f67448e89..3682a806ac9 100644 --- a/dbms/include/DB/Functions/FunctionsMiscellaneous.h +++ b/dbms/include/DB/Functions/FunctionsMiscellaneous.h @@ -3,7 +3,6 @@ #include #include -#include #include #include @@ -22,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -57,8 +55,6 @@ namespace DB * sleep(n) - спит n секунд каждый блок. * * bar(x, min, max, width) - рисует полосу из количества символов, пропорционального (x - min) и равного width при x == max. - * - * transform(x, from_array, to_array[, default]) - преобразовать x согласно переданному явным образом соответствию. */ @@ -892,505 +888,4 @@ using FunctionIsInfinite = FunctionNumericPredicate; using FunctionIsNaN = FunctionNumericPredicate; -DataTypePtr getSmallestCommonNumericType(const IDataType & t1, const IDataType & t2); - -/** transform(x, [from...], [to...], default) - * - преобразует значения согласно явно указанному отображению. - * - * x - что преобразовывать. - * from - константный массив значений для преобразования. - * to - константный массив значений, в которые должны быть преобразованы значения из from. - * default - константа, какое значение использовать, если x не равен ни одному из значений во from. - * from и to - массивы одинаковых размеров. - * - * Типы: - * transform(T, Array(T), Array(U), U) -> U - * - * transform(x, [from...], [to...]) - * - eсли default не указан, то для значений x, для которых нет соответствующего элемента во from, возвращается не изменённое значение x. - * - * Типы: - * transform(T, Array(T), Array(T)) -> T - * - * Замечание: реализация довольно громоздкая. - */ -class FunctionTransform : public IFunction -{ -public: - static constexpr auto name = "transform"; - static IFunction * create(const Context &) { return new FunctionTransform; } - - String getName() const override { return name; } - - DataTypePtr getReturnType(const DataTypes & arguments) const override - { - const auto args_size = arguments.size(); - if (args_size != 3 && args_size != 4) - throw Exception{ - "Number of arguments for function " + getName() + " doesn't match: passed " + - toString(args_size) + ", should be 3 or 4", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; - - const IDataType * type_x = arguments[0].get(); - - if (!type_x->isNumeric() && !typeid_cast(type_x)) - throw Exception("Unsupported type " + type_x->getName() - + " of first argument of function " + getName() - + ", must be numeric type or Date/DateTime or String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - const DataTypeArray * type_arr_from = typeid_cast(arguments[1].get()); - - if (!type_arr_from) - throw Exception("Second argument of function " + getName() - + ", must be array of source values to transform from.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - const auto type_arr_from_nested = type_arr_from->getNestedType(); - - if ((type_x->isNumeric() != type_arr_from_nested->isNumeric()) - || (!!typeid_cast(type_x) != !!typeid_cast(type_arr_from_nested.get()))) - throw Exception("First argument and elements of array of second argument of function " + getName() - + " must have compatible types: both numeric or both strings.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - const DataTypeArray * type_arr_to = typeid_cast(arguments[2].get()); - - if (!type_arr_to) - throw Exception("Third argument of function " + getName() - + ", must be array of destination values to transform to.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - const auto type_arr_to_nested = type_arr_to->getNestedType(); - - if (args_size == 3) - { - if ((type_x->isNumeric() != type_arr_to_nested->isNumeric()) - || (!!typeid_cast(type_x) != !!typeid_cast(type_arr_to_nested.get()))) - throw Exception("Function " + getName() - + " have signature: transform(T, Array(T), Array(U), U) -> U; or transform(T, Array(T), Array(T)) -> T; where T and U are types.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - return type_x->clone(); - } - else - { - const IDataType * type_default = arguments[3].get(); - - if (!type_default->isNumeric() && !typeid_cast(type_default)) - throw Exception("Unsupported type " + type_default->getName() - + " of fourth argument (default value) of function " + getName() - + ", must be numeric type or Date/DateTime or String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if ((type_default->isNumeric() != type_arr_to_nested->isNumeric()) - || (!!typeid_cast(type_default) != !!typeid_cast(type_arr_to_nested.get()))) - throw Exception("Function " + getName() - + " have signature: transform(T, Array(T), Array(U), U) -> U; or transform(T, Array(T), Array(T)) -> T; where T and U are types.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - if (type_arr_to_nested->behavesAsNumber() && type_default->behavesAsNumber()) - { - /// Берём наименьший общий тип для элементов массива значений to и для default-а. - return getSmallestCommonNumericType(*type_arr_to_nested, *type_default); - } - - /// TODO Больше проверок. - return type_arr_to_nested->clone(); - } - } - - void execute(Block & block, const ColumnNumbers & arguments, const size_t result) override - { - const ColumnConstArray * array_from = typeid_cast(&*block.getByPosition(arguments[1]).column); - const ColumnConstArray * array_to = typeid_cast(&*block.getByPosition(arguments[2]).column); - - if (!array_from && !array_to) - throw Exception("Second and third arguments of function " + getName() + " must be constant arrays.", ErrorCodes::ILLEGAL_COLUMN); - - prepare(array_from->getData(), array_to->getData(), block, arguments); - - const auto in = block.getByPosition(arguments.front()).column.get(); - - if (in->isConst()) - { - executeConst(block, arguments, result); - return; - } - - auto column_result = block.getByPosition(result).type->createColumn(); - auto out = column_result.get(); - - if (!executeNum(in, out) - && !executeNum(in, out) - && !executeNum(in, out) - && !executeNum(in, out) - && !executeNum(in, out) - && !executeNum(in, out) - && !executeNum(in, out) - && !executeNum(in, out) - && !executeNum(in, out) - && !executeNum(in, out) - && !executeString(in, out)) - throw Exception( - "Illegal column " + in->getName() + " of first argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - - block.getByPosition(result).column = column_result; - } - -private: - void executeConst(Block & block, const ColumnNumbers & arguments, const size_t result) - { - /// Составим блок из полноценных столбцов размера 1 и вычислим функцию как обычно. - - Block tmp_block; - ColumnNumbers tmp_arguments; - - tmp_block.insert(block.getByPosition(arguments[0])); - tmp_block.getByPosition(0).column = static_cast(tmp_block.getByPosition(0).column->cloneResized(1).get())->convertToFullColumn(); - tmp_arguments.push_back(0); - - for (size_t i = 1; i < arguments.size(); ++i) - { - tmp_block.insert(block.getByPosition(arguments[i])); - tmp_arguments.push_back(i); - } - - tmp_block.insert(block.getByPosition(result)); - size_t tmp_result = arguments.size(); - - execute(tmp_block, tmp_arguments, tmp_result); - - block.getByPosition(result).column = block.getByPosition(result).type->createConstColumn( - block.rowsInFirstColumn(), - (*tmp_block.getByPosition(tmp_result).column)[0]); - } - - template - bool executeNum(const IColumn * in_untyped, IColumn * out_untyped) - { - if (const auto in = typeid_cast *>(in_untyped)) - { - if (default_value.isNull()) - { - auto out = typeid_cast *>(out_untyped); - if (!out) - throw Exception( - "Illegal column " + out_untyped->getName() + " of elements of array of third argument of function " + getName() - + ", must be " + in->getName(), - ErrorCodes::ILLEGAL_COLUMN); - - executeImplNumToNum(in->getData(), out->getData()); - } - else - { - if (!executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToNumWithDefault(in, out_untyped) - && !executeNumToString(in, out_untyped)) - throw Exception( - "Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - } - - return true; - } - - return false; - } - - bool executeString(const IColumn * in_untyped, IColumn * out_untyped) - { - if (const auto in = typeid_cast(in_untyped)) - { - if (!executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToNum(in, out_untyped) - && !executeStringToString(in, out_untyped)) - throw Exception( - "Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(), - ErrorCodes::ILLEGAL_COLUMN); - - return true; - } - - return false; - } - - template - bool executeNumToNumWithDefault(const ColumnVector * in, IColumn * out_untyped) - { - auto out = typeid_cast *>(out_untyped); - if (!out) - return false; - - executeImplNumToNumWithDefault(in->getData(), out->getData(), default_value.get()); - return true; - } - - template - bool executeNumToString(const ColumnVector * in, IColumn * out_untyped) - { - auto out = typeid_cast(out_untyped); - if (!out) - return false; - - const String & default_str = default_value.get(); - StringRef default_string_ref{default_str.data(), default_str.size() + 1}; - executeImplNumToStringWithDefault(in->getData(), out->getChars(), out->getOffsets(), default_string_ref); - return true; - } - - template - bool executeStringToNum(const ColumnString * in, IColumn * out_untyped) - { - auto out = typeid_cast *>(out_untyped); - if (!out) - return false; - - executeImplStringToNumWithDefault(in->getChars(), in->getOffsets(), out->getData(), default_value.get()); - return true; - } - - bool executeStringToString(const ColumnString * in, IColumn * out_untyped) - { - auto out = typeid_cast(out_untyped); - if (!out) - return false; - - if (default_value.isNull()) - executeImplStringToString(in->getChars(), in->getOffsets(), out->getChars(), out->getOffsets(), {}); - else - { - const String & default_str = default_value.get(); - StringRef default_string_ref{default_str.data(), default_str.size() + 1}; - executeImplStringToString(in->getChars(), in->getOffsets(), out->getChars(), out->getOffsets(), default_string_ref); - } - - return true; - } - - - template - void executeImplNumToNumWithDefault(const PODArray & src, PODArray & dst, U dst_default) - { - const auto & table = *table_num_to_num; - size_t size = src.size(); - dst.resize(size); - for (size_t i = 0; i < size; ++i) - { - auto it = table.find(src[i]); - if (it != table.end()) - memcpy(&dst[i], &it->second, sizeof(dst[i])); /// little endian. - else - dst[i] = dst_default; - } - } - - template - void executeImplNumToNum(const PODArray & src, PODArray & dst) - { - const auto & table = *table_num_to_num; - size_t size = src.size(); - dst.resize(size); - for (size_t i = 0; i < size; ++i) - { - auto it = table.find(src[i]); - if (it != table.end()) - memcpy(&dst[i], &it->second, sizeof(dst[i])); - else - dst[i] = src[i]; - } - } - - template - void executeImplNumToStringWithDefault(const PODArray & src, - ColumnString::Chars_t & dst_data, ColumnString::Offsets_t & dst_offsets, StringRef dst_default) - { - const auto & table = *table_num_to_string; - size_t size = src.size(); - dst_offsets.resize(size); - ColumnString::Offset_t current_offset = 0; - for (size_t i = 0; i < size; ++i) - { - auto it = table.find(src[i]); - StringRef ref = it != table.end() ? it->second : dst_default; - dst_data.resize(current_offset + ref.size); - memcpy(&dst_data[current_offset], ref.data, ref.size); - current_offset += ref.size; - dst_offsets[i] = current_offset; - } - } - - template - void executeImplStringToNumWithDefault( - const ColumnString::Chars_t & src_data, const ColumnString::Offsets_t & src_offsets, - PODArray & dst, U dst_default) - { - const auto & table = *table_string_to_num; - size_t size = src_offsets.size(); - dst.resize(size); - ColumnString::Offset_t current_offset = 0; - for (size_t i = 0; i < size; ++i) - { - StringRef ref{&src_data[current_offset], src_offsets[i] - current_offset}; - current_offset = src_offsets[i]; - auto it = table.find(ref); - if (it != table.end()) - memcpy(&dst[i], &it->second, sizeof(dst[i])); - else - dst[i] = dst_default; - } - } - - template - void executeImplStringToString( - const ColumnString::Chars_t & src_data, const ColumnString::Offsets_t & src_offsets, - ColumnString::Chars_t & dst_data, ColumnString::Offsets_t & dst_offsets, StringRef dst_default) - { - const auto & table = *table_string_to_string; - size_t size = src_offsets.size(); - dst_offsets.resize(size); - ColumnString::Offset_t current_src_offset = 0; - ColumnString::Offset_t current_dst_offset = 0; - for (size_t i = 0; i < size; ++i) - { - StringRef src_ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; - current_src_offset = src_offsets[i]; - - auto it = table.find(src_ref); - - StringRef dst_ref = it != table.end() ? it->second : (with_default ? dst_default : src_ref); - dst_data.resize(current_dst_offset + dst_ref.size); - memcpy(&dst_data[current_dst_offset], dst_ref.data, dst_ref.size); - current_dst_offset += dst_ref.size; - dst_offsets[i] = current_dst_offset; - } - } - - - /// Разные варианты хэш-таблиц для реализации отображения. - - using NumToNum = HashMap>; - using NumToString = HashMap>; /// Везде StringRef-ы с завершающим нулём. - using StringToNum = HashMap; - using StringToString = HashMap; - - std::unique_ptr table_num_to_num; - std::unique_ptr table_num_to_string; - std::unique_ptr table_string_to_num; - std::unique_ptr table_string_to_string; - - Arena string_pool; - - Field default_value; /// Null, если не задано. - - bool prepared = false; - std::mutex mutex; - - /// Может вызываться из разных потоков. Срабатывает только при первом вызове. - void prepare(const Array & from, const Array & to, Block & block, const ColumnNumbers & arguments) - { - if (prepared) - return; - - const size_t size = from.size(); - if (0 == size) - throw Exception("Empty arrays are illegal in function " + getName(), ErrorCodes::BAD_ARGUMENTS); - - std::lock_guard lock(mutex); - - if (prepared) - return; - - if (from.size() != to.size()) - throw Exception("Second and third arguments of function " + getName() + " must be arrays of same size", ErrorCodes::BAD_ARGUMENTS); - - Array converted_to; - const Array * used_to = &to; - - /// Задано ли значение по-умолчанию. - - if (arguments.size() == 4) - { - const IColumnConst * default_col = dynamic_cast(&*block.getByPosition(arguments[3]).column); - - if (!default_col) - throw Exception("Fourth argument of function " + getName() + " (default value) must be constant", ErrorCodes::ILLEGAL_COLUMN); - - default_value = (*default_col)[0]; - - /// Нужно ли преобразовать элементы to и default_value к наименьшему общему типу, который является Float64? - if (default_value.getType() == Field::Types::Float64 && to[0].getType() != Field::Types::Float64) - { - converted_to.resize(to.size()); - for (size_t i = 0, size = to.size(); i < size; ++i) - converted_to[i] = apply_visitor(FieldVisitorConvertToNumber(), to[i]); - used_to = &converted_to; - } - else if (default_value.getType() != Field::Types::Float64 && to[0].getType() == Field::Types::Float64) - { - default_value = apply_visitor(FieldVisitorConvertToNumber(), default_value); - } - } - - /// Замечание: не делается проверка дубликатов в массиве from. - - if (from[0].getType() != Field::Types::String && to[0].getType() != Field::Types::String) - { - table_num_to_num.reset(new NumToNum); - auto & table = *table_num_to_num; - for (size_t i = 0; i < size; ++i) - table[from[i].get()] = (*used_to)[i].get(); - } - else if (from[0].getType() != Field::Types::String && to[0].getType() == Field::Types::String) - { - table_num_to_string.reset(new NumToString); - auto & table = *table_num_to_string; - for (size_t i = 0; i < size; ++i) - { - const String & str_to = to[i].get(); - StringRef ref{string_pool.insert(str_to.data(), str_to.size() + 1), str_to.size() + 1}; - table[from[i].get()] = ref; - } - } - else if (from[0].getType() == Field::Types::String && to[0].getType() != Field::Types::String) - { - table_string_to_num.reset(new StringToNum); - auto & table = *table_string_to_num; - for (size_t i = 0; i < size; ++i) - { - const String & str_from = from[i].get(); - StringRef ref{string_pool.insert(str_from.data(), str_from.size() + 1), str_from.size() + 1}; - table[ref] = (*used_to)[i].get(); - } - } - else if (from[0].getType() == Field::Types::String && to[0].getType() == Field::Types::String) - { - table_string_to_string.reset(new StringToString); - auto & table = *table_string_to_string; - for (size_t i = 0; i < size; ++i) - { - const String & str_from = from[i].get(); - const String & str_to = to[i].get(); - StringRef ref_from{string_pool.insert(str_from.data(), str_from.size() + 1), str_from.size() + 1}; - StringRef ref_to{string_pool.insert(str_to.data(), str_to.size() + 1), str_to.size() + 1}; - table[ref_from] = ref_to; - } - } - - prepared = true; - } -}; - } diff --git a/dbms/include/DB/Functions/FunctionsTransform.h b/dbms/include/DB/Functions/FunctionsTransform.h new file mode 100644 index 00000000000..b1b3a2ce79a --- /dev/null +++ b/dbms/include/DB/Functions/FunctionsTransform.h @@ -0,0 +1,804 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +/** transform(x, from_array, to_array[, default]) - преобразовать x согласно переданному явным образом соответствию. + */ + +DataTypePtr getSmallestCommonNumericType(const IDataType & t1, const IDataType & t2); + +/** transform(x, [from...], [to...], default) + * - преобразует значения согласно явно указанному отображению. + * + * x - что преобразовывать. + * from - константный массив значений для преобразования. + * to - константный массив значений, в которые должны быть преобразованы значения из from. + * default - какое значение использовать, если x не равен ни одному из значений во from. + * from и to - массивы одинаковых размеров. + * + * Типы: + * transform(T, Array(T), Array(U), U) -> U + * + * transform(x, [from...], [to...]) + * - eсли default не указан, то для значений x, для которых нет соответствующего элемента во from, возвращается не изменённое значение x. + * + * Типы: + * transform(T, Array(T), Array(T)) -> T + * + * Замечание: реализация довольно громоздкая. + */ +class FunctionTransform : public IFunction +{ +public: + static constexpr auto name = "transform"; + static IFunction * create(const Context &) { return new FunctionTransform; } + + String getName() const override { return name; } + + DataTypePtr getReturnType(const DataTypes & arguments) const override + { + const auto args_size = arguments.size(); + if (args_size != 3 && args_size != 4) + throw Exception{ + "Number of arguments for function " + getName() + " doesn't match: passed " + + toString(args_size) + ", should be 3 or 4", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH}; + + const IDataType * type_x = arguments[0].get(); + + if (!type_x->isNumeric() && !typeid_cast(type_x)) + throw Exception("Unsupported type " + type_x->getName() + + " of first argument of function " + getName() + + ", must be numeric type or Date/DateTime or String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const DataTypeArray * type_arr_from = typeid_cast(arguments[1].get()); + + if (!type_arr_from) + throw Exception("Second argument of function " + getName() + + ", must be array of source values to transform from.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const auto type_arr_from_nested = type_arr_from->getNestedType(); + + if ((type_x->isNumeric() != type_arr_from_nested->isNumeric()) + || (!!typeid_cast(type_x) != !!typeid_cast(type_arr_from_nested.get()))) + throw Exception("First argument and elements of array of second argument of function " + getName() + + " must have compatible types: both numeric or both strings.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const DataTypeArray * type_arr_to = typeid_cast(arguments[2].get()); + + if (!type_arr_to) + throw Exception("Third argument of function " + getName() + + ", must be array of destination values to transform to.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + const auto type_arr_to_nested = type_arr_to->getNestedType(); + + if (args_size == 3) + { + if ((type_x->isNumeric() != type_arr_to_nested->isNumeric()) + || (!!typeid_cast(type_x) != !!typeid_cast(type_arr_to_nested.get()))) + throw Exception("Function " + getName() + + " have signature: transform(T, Array(T), Array(U), U) -> U; or transform(T, Array(T), Array(T)) -> T; where T and U are types.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return type_x->clone(); + } + else + { + const IDataType * type_default = arguments[3].get(); + + if (!type_default->isNumeric() && !typeid_cast(type_default)) + throw Exception("Unsupported type " + type_default->getName() + + " of fourth argument (default value) of function " + getName() + + ", must be numeric type or Date/DateTime or String", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if ((type_default->isNumeric() != type_arr_to_nested->isNumeric()) + || (!!typeid_cast(type_default) != !!typeid_cast(type_arr_to_nested.get()))) + throw Exception("Function " + getName() + + " have signature: transform(T, Array(T), Array(U), U) -> U; or transform(T, Array(T), Array(T)) -> T; where T and U are types.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (type_arr_to_nested->behavesAsNumber() && type_default->behavesAsNumber()) + { + /// Берём наименьший общий тип для элементов массива значений to и для default-а. + return getSmallestCommonNumericType(*type_arr_to_nested, *type_default); + } + + /// TODO Больше проверок. + return type_arr_to_nested->clone(); + } + } + + void execute(Block & block, const ColumnNumbers & arguments, const size_t result) override + { + const ColumnConstArray * array_from = typeid_cast(&*block.getByPosition(arguments[1]).column); + const ColumnConstArray * array_to = typeid_cast(&*block.getByPosition(arguments[2]).column); + + if (!array_from && !array_to) + throw Exception("Second and third arguments of function " + getName() + " must be constant arrays.", ErrorCodes::ILLEGAL_COLUMN); + + prepare(array_from->getData(), array_to->getData(), block, arguments); + + const auto in = block.getByPosition(arguments.front()).column.get(); + + if (in->isConst()) + { + executeConst(block, arguments, result); + return; + } + + const IColumn * default_column = nullptr; + if (arguments.size() == 4) + default_column = block.getByPosition(arguments[3]).column.get(); + + auto column_result = block.getByPosition(result).type->createColumn(); + auto out = column_result.get(); + + if (!executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeNum(in, out, default_column) + && !executeString(in, out, default_column)) + throw Exception( + "Illegal column " + in->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + block.getByPosition(result).column = column_result; + } + +private: + void executeConst(Block & block, const ColumnNumbers & arguments, const size_t result) + { + /// Составим блок из полноценных столбцов размера 1 и вычислим функцию как обычно. + + Block tmp_block; + ColumnNumbers tmp_arguments; + + tmp_block.insert(block.getByPosition(arguments[0])); + tmp_block.getByPosition(0).column = static_cast(tmp_block.getByPosition(0).column->cloneResized(1).get())->convertToFullColumn(); + tmp_arguments.push_back(0); + + for (size_t i = 1; i < arguments.size(); ++i) + { + tmp_block.insert(block.getByPosition(arguments[i])); + tmp_arguments.push_back(i); + } + + tmp_block.insert(block.getByPosition(result)); + size_t tmp_result = arguments.size(); + + execute(tmp_block, tmp_arguments, tmp_result); + + block.getByPosition(result).column = block.getByPosition(result).type->createConstColumn( + block.rowsInFirstColumn(), + (*tmp_block.getByPosition(tmp_result).column)[0]); + } + + template + bool executeNum(const IColumn * in_untyped, IColumn * out_untyped, const IColumn * default_untyped) + { + if (const auto in = typeid_cast *>(in_untyped)) + { + if (!default_untyped) + { + auto out = typeid_cast *>(out_untyped); + if (!out) + throw Exception( + "Illegal column " + out_untyped->getName() + " of elements of array of third argument of function " + getName() + + ", must be " + in->getName(), + ErrorCodes::ILLEGAL_COLUMN); + + executeImplNumToNum(in->getData(), out->getData()); + } + else if (default_untyped->isConst()) + { + if (!executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToNumWithConstDefault(in, out_untyped) + && !executeNumToStringWithConstDefault(in, out_untyped)) + throw Exception( + "Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + else + { + if (!executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeNumToStringWithNonConstDefault(in, out_untyped, default_untyped)) + throw Exception( + "Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + + return true; + } + + return false; + } + + bool executeString(const IColumn * in_untyped, IColumn * out_untyped, const IColumn * default_untyped) + { + if (const auto in = typeid_cast(in_untyped)) + { + if (!default_untyped) + { + if (!executeStringToString(in, out_untyped)) + throw Exception( + "Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + else if (default_untyped->isConst()) + { + if (!executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToNumWithConstDefault(in, out_untyped) + && !executeStringToStringWithConstDefault(in, out_untyped)) + throw Exception( + "Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + else + { + if (!executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToNumWithNonConstDefault(in, out_untyped, default_untyped) + && !executeStringToStringWithNonConstDefault(in, out_untyped, default_untyped)) + throw Exception( + "Illegal column " + in->getName() + " of elements of array of second argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + } + + return true; + } + + return false; + } + + template + bool executeNumToNumWithConstDefault(const ColumnVector * in, IColumn * out_untyped) + { + auto out = typeid_cast *>(out_untyped); + if (!out) + return false; + + executeImplNumToNumWithConstDefault(in->getData(), out->getData(), const_default_value.get()); + return true; + } + + template + bool executeNumToNumWithNonConstDefault(const ColumnVector * in, IColumn * out_untyped, const IColumn * default_untyped) + { + auto out = typeid_cast *>(out_untyped); + if (!out) + return false; + + if (!executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped) + && !executeNumToNumWithNonConstDefault2(in, out, default_untyped)) + throw Exception( + "Illegal column " + default_untyped->getName() + " of fourth argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + return true; + } + + template + bool executeNumToNumWithNonConstDefault2(const ColumnVector * in, ColumnVector * out, const IColumn * default_untyped) + { + auto col_default = typeid_cast *>(default_untyped); + if (!col_default) + return false; + + executeImplNumToNumWithNonConstDefault(in->getData(), out->getData(), col_default->getData()); + return true; + } + + template + bool executeNumToStringWithConstDefault(const ColumnVector * in, IColumn * out_untyped) + { + auto out = typeid_cast(out_untyped); + if (!out) + return false; + + const String & default_str = const_default_value.get(); + StringRef default_string_ref{default_str.data(), default_str.size() + 1}; + executeImplNumToStringWithConstDefault(in->getData(), out->getChars(), out->getOffsets(), default_string_ref); + return true; + } + + template + bool executeNumToStringWithNonConstDefault(const ColumnVector * in, IColumn * out_untyped, const IColumn * default_untyped) + { + auto out = typeid_cast(out_untyped); + if (!out) + return false; + + auto default_col = typeid_cast(default_untyped); + if (!default_col) + throw Exception("Illegal column " + default_untyped->getName() + " of fourth argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + executeImplNumToStringWithNonConstDefault( + in->getData(), + out->getChars(), out->getOffsets(), + default_col->getChars(), default_col->getOffsets()); + + return true; + } + + template + bool executeStringToNumWithConstDefault(const ColumnString * in, IColumn * out_untyped) + { + auto out = typeid_cast *>(out_untyped); + if (!out) + return false; + + executeImplStringToNumWithConstDefault(in->getChars(), in->getOffsets(), out->getData(), const_default_value.get()); + return true; + } + + template + bool executeStringToNumWithNonConstDefault(const ColumnString * in, IColumn * out_untyped, const IColumn * default_untyped) + { + auto out = typeid_cast *>(out_untyped); + if (!out) + return false; + + if (!executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped) + && !executeStringToNumWithNonConstDefault2(in, out, default_untyped)) + throw Exception( + "Illegal column " + default_untyped->getName() + " of fourth argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + return true; + } + + template + bool executeStringToNumWithNonConstDefault2(const ColumnString * in, ColumnVector * out, const IColumn * default_untyped) + { + auto col_default = typeid_cast *>(default_untyped); + if (!col_default) + return false; + + executeImplStringToNumWithNonConstDefault(in->getChars(), in->getOffsets(), out->getData(), col_default->getData()); + return true; + } + + bool executeStringToString(const ColumnString * in, IColumn * out_untyped) + { + auto out = typeid_cast(out_untyped); + if (!out) + return false; + + executeImplStringToString(in->getChars(), in->getOffsets(), out->getChars(), out->getOffsets()); + return true; + } + + bool executeStringToStringWithConstDefault(const ColumnString * in, IColumn * out_untyped) + { + auto out = typeid_cast(out_untyped); + if (!out) + return false; + + const String & default_str = const_default_value.get(); + StringRef default_string_ref{default_str.data(), default_str.size() + 1}; + executeImplStringToStringWithConstDefault(in->getChars(), in->getOffsets(), out->getChars(), out->getOffsets(), default_string_ref); + return true; + } + + bool executeStringToStringWithNonConstDefault(const ColumnString * in, IColumn * out_untyped, const IColumn * default_untyped) + { + auto out = typeid_cast(out_untyped); + if (!out) + return false; + + auto default_col = typeid_cast(default_untyped); + if (!default_col) + throw Exception("Illegal column " + default_untyped->getName() + " of fourth argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + executeImplStringToStringWithNonConstDefault( + in->getChars(), in->getOffsets(), + out->getChars(), out->getOffsets(), + default_col->getChars(), default_col->getOffsets()); + + return true; + } + + + template + void executeImplNumToNumWithConstDefault(const PODArray & src, PODArray & dst, U dst_default) + { + const auto & table = *table_num_to_num; + size_t size = src.size(); + dst.resize(size); + for (size_t i = 0; i < size; ++i) + { + auto it = table.find(src[i]); + if (it != table.end()) + memcpy(&dst[i], &it->second, sizeof(dst[i])); /// little endian. + else + dst[i] = dst_default; + } + } + + template + void executeImplNumToNumWithNonConstDefault(const PODArray & src, PODArray & dst, const PODArray & dst_default) + { + const auto & table = *table_num_to_num; + size_t size = src.size(); + dst.resize(size); + for (size_t i = 0; i < size; ++i) + { + auto it = table.find(src[i]); + if (it != table.end()) + memcpy(&dst[i], &it->second, sizeof(dst[i])); /// little endian. + else + dst[i] = dst_default[i]; + } + } + + template + void executeImplNumToNum(const PODArray & src, PODArray & dst) + { + const auto & table = *table_num_to_num; + size_t size = src.size(); + dst.resize(size); + for (size_t i = 0; i < size; ++i) + { + auto it = table.find(src[i]); + if (it != table.end()) + memcpy(&dst[i], &it->second, sizeof(dst[i])); + else + dst[i] = src[i]; + } + } + + template + void executeImplNumToStringWithConstDefault(const PODArray & src, + ColumnString::Chars_t & dst_data, ColumnString::Offsets_t & dst_offsets, StringRef dst_default) + { + const auto & table = *table_num_to_string; + size_t size = src.size(); + dst_offsets.resize(size); + ColumnString::Offset_t current_dst_offset = 0; + for (size_t i = 0; i < size; ++i) + { + auto it = table.find(src[i]); + StringRef ref = it != table.end() ? it->second : dst_default; + dst_data.resize(current_dst_offset + ref.size); + memcpy(&dst_data[current_dst_offset], ref.data, ref.size); + current_dst_offset += ref.size; + dst_offsets[i] = current_dst_offset; + } + } + + template + void executeImplNumToStringWithNonConstDefault(const PODArray & src, + ColumnString::Chars_t & dst_data, ColumnString::Offsets_t & dst_offsets, + const ColumnString::Chars_t & dst_default_data, const ColumnString::Offsets_t & dst_default_offsets) + { + const auto & table = *table_num_to_string; + size_t size = src.size(); + dst_offsets.resize(size); + ColumnString::Offset_t current_dst_offset = 0; + ColumnString::Offset_t current_dst_default_offset = 0; + for (size_t i = 0; i < size; ++i) + { + auto it = table.find(src[i]); + StringRef ref; + + if (it != table.end()) + ref = it->second; + else + { + ref.data = reinterpret_cast(&dst_default_data[current_dst_default_offset]); + ref.size = dst_default_offsets[i] - current_dst_default_offset; + } + + dst_data.resize(current_dst_offset + ref.size); + memcpy(&dst_data[current_dst_offset], ref.data, ref.size); + current_dst_offset += ref.size; + current_dst_default_offset = dst_default_offsets[i]; + dst_offsets[i] = current_dst_offset; + } + } + + template + void executeImplStringToNumWithConstDefault( + const ColumnString::Chars_t & src_data, const ColumnString::Offsets_t & src_offsets, + PODArray & dst, U dst_default) + { + const auto & table = *table_string_to_num; + size_t size = src_offsets.size(); + dst.resize(size); + ColumnString::Offset_t current_src_offset = 0; + for (size_t i = 0; i < size; ++i) + { + StringRef ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; + current_src_offset = src_offsets[i]; + auto it = table.find(ref); + if (it != table.end()) + memcpy(&dst[i], &it->second, sizeof(dst[i])); + else + dst[i] = dst_default; + } + } + + template + void executeImplStringToNumWithNonConstDefault( + const ColumnString::Chars_t & src_data, const ColumnString::Offsets_t & src_offsets, + PODArray & dst, const PODArray & dst_default) + { + const auto & table = *table_string_to_num; + size_t size = src_offsets.size(); + dst.resize(size); + ColumnString::Offset_t current_src_offset = 0; + for (size_t i = 0; i < size; ++i) + { + StringRef ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; + current_src_offset = src_offsets[i]; + auto it = table.find(ref); + if (it != table.end()) + memcpy(&dst[i], &it->second, sizeof(dst[i])); + else + dst[i] = dst_default[i]; + } + } + + template + void executeImplStringToStringWithOrWithoutConstDefault( + const ColumnString::Chars_t & src_data, const ColumnString::Offsets_t & src_offsets, + ColumnString::Chars_t & dst_data, ColumnString::Offsets_t & dst_offsets, StringRef dst_default) + { + const auto & table = *table_string_to_string; + size_t size = src_offsets.size(); + dst_offsets.resize(size); + ColumnString::Offset_t current_src_offset = 0; + ColumnString::Offset_t current_dst_offset = 0; + for (size_t i = 0; i < size; ++i) + { + StringRef src_ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; + current_src_offset = src_offsets[i]; + + auto it = table.find(src_ref); + + StringRef dst_ref = it != table.end() ? it->second : (with_default ? dst_default : src_ref); + dst_data.resize(current_dst_offset + dst_ref.size); + memcpy(&dst_data[current_dst_offset], dst_ref.data, dst_ref.size); + current_dst_offset += dst_ref.size; + dst_offsets[i] = current_dst_offset; + } + } + + void executeImplStringToString( + const ColumnString::Chars_t & src_data, const ColumnString::Offsets_t & src_offsets, + ColumnString::Chars_t & dst_data, ColumnString::Offsets_t & dst_offsets) + { + executeImplStringToStringWithOrWithoutConstDefault(src_data, src_offsets, dst_data, dst_offsets, {}); + } + + void executeImplStringToStringWithConstDefault( + const ColumnString::Chars_t & src_data, const ColumnString::Offsets_t & src_offsets, + ColumnString::Chars_t & dst_data, ColumnString::Offsets_t & dst_offsets, StringRef dst_default) + { + executeImplStringToStringWithOrWithoutConstDefault(src_data, src_offsets, dst_data, dst_offsets, dst_default); + } + + void executeImplStringToStringWithNonConstDefault( + const ColumnString::Chars_t & src_data, const ColumnString::Offsets_t & src_offsets, + ColumnString::Chars_t & dst_data, ColumnString::Offsets_t & dst_offsets, + const ColumnString::Chars_t & dst_default_data, const ColumnString::Offsets_t & dst_default_offsets) + { + const auto & table = *table_string_to_string; + size_t size = src_offsets.size(); + dst_offsets.resize(size); + ColumnString::Offset_t current_src_offset = 0; + ColumnString::Offset_t current_dst_offset = 0; + ColumnString::Offset_t current_dst_default_offset = 0; + for (size_t i = 0; i < size; ++i) + { + StringRef src_ref{&src_data[current_src_offset], src_offsets[i] - current_src_offset}; + current_src_offset = src_offsets[i]; + + auto it = table.find(src_ref); + StringRef dst_ref; + + if (it != table.end()) + dst_ref = it->second; + else + { + dst_ref.data = reinterpret_cast(&dst_default_data[current_dst_default_offset]); + dst_ref.size = dst_default_offsets[i] - current_dst_default_offset; + } + + dst_data.resize(current_dst_offset + dst_ref.size); + memcpy(&dst_data[current_dst_offset], dst_ref.data, dst_ref.size); + current_dst_offset += dst_ref.size; + current_dst_default_offset = dst_default_offsets[i]; + dst_offsets[i] = current_dst_offset; + } + } + + + /// Разные варианты хэш-таблиц для реализации отображения. + + using NumToNum = HashMap>; + using NumToString = HashMap>; /// Везде StringRef-ы с завершающим нулём. + using StringToNum = HashMap; + using StringToString = HashMap; + + std::unique_ptr table_num_to_num; + std::unique_ptr table_num_to_string; + std::unique_ptr table_string_to_num; + std::unique_ptr table_string_to_string; + + Arena string_pool; + + Field const_default_value; /// Null, если не задано. + + bool prepared = false; + std::mutex mutex; + + /// Может вызываться из разных потоков. Срабатывает только при первом вызове. + void prepare(const Array & from, const Array & to, Block & block, const ColumnNumbers & arguments) + { + if (prepared) + return; + + const size_t size = from.size(); + if (0 == size) + throw Exception("Empty arrays are illegal in function " + getName(), ErrorCodes::BAD_ARGUMENTS); + + std::lock_guard lock(mutex); + + if (prepared) + return; + + if (from.size() != to.size()) + throw Exception("Second and third arguments of function " + getName() + " must be arrays of same size", ErrorCodes::BAD_ARGUMENTS); + + Array converted_to; + const Array * used_to = &to; + + /// Задано ли значение по-умолчанию. + + if (arguments.size() == 4) + { + const IColumnConst * default_col = dynamic_cast(&*block.getByPosition(arguments[3]).column); + + if (default_col) + { + const_default_value = (*default_col)[0]; + + /// Нужно ли преобразовать элементы to и default_value к наименьшему общему типу, который является Float64? + if (const_default_value.getType() == Field::Types::Float64 && to[0].getType() != Field::Types::Float64) + { + converted_to.resize(to.size()); + for (size_t i = 0, size = to.size(); i < size; ++i) + converted_to[i] = apply_visitor(FieldVisitorConvertToNumber(), to[i]); + used_to = &converted_to; + } + else if (const_default_value.getType() != Field::Types::Float64 && to[0].getType() == Field::Types::Float64) + { + const_default_value = apply_visitor(FieldVisitorConvertToNumber(), const_default_value); + } + } + + /// TODO Преобразование для to в случае неконстантного default + } + + /// Замечание: не делается проверка дубликатов в массиве from. + + if (from[0].getType() != Field::Types::String && to[0].getType() != Field::Types::String) + { + table_num_to_num.reset(new NumToNum); + auto & table = *table_num_to_num; + for (size_t i = 0; i < size; ++i) + table[from[i].get()] = (*used_to)[i].get(); + } + else if (from[0].getType() != Field::Types::String && to[0].getType() == Field::Types::String) + { + table_num_to_string.reset(new NumToString); + auto & table = *table_num_to_string; + for (size_t i = 0; i < size; ++i) + { + const String & str_to = to[i].get(); + StringRef ref{string_pool.insert(str_to.data(), str_to.size() + 1), str_to.size() + 1}; + table[from[i].get()] = ref; + } + } + else if (from[0].getType() == Field::Types::String && to[0].getType() != Field::Types::String) + { + table_string_to_num.reset(new StringToNum); + auto & table = *table_string_to_num; + for (size_t i = 0; i < size; ++i) + { + const String & str_from = from[i].get(); + StringRef ref{string_pool.insert(str_from.data(), str_from.size() + 1), str_from.size() + 1}; + table[ref] = (*used_to)[i].get(); + } + } + else if (from[0].getType() == Field::Types::String && to[0].getType() == Field::Types::String) + { + table_string_to_string.reset(new StringToString); + auto & table = *table_string_to_string; + for (size_t i = 0; i < size; ++i) + { + const String & str_from = from[i].get(); + const String & str_to = to[i].get(); + StringRef ref_from{string_pool.insert(str_from.data(), str_from.size() + 1), str_from.size() + 1}; + StringRef ref_to{string_pool.insert(str_to.data(), str_to.size() + 1), str_to.size() + 1}; + table[ref_from] = ref_to; + } + } + + prepared = true; + } +}; + +} diff --git a/dbms/src/DataStreams/VerticalRowOutputStream.cpp b/dbms/src/DataStreams/VerticalRowOutputStream.cpp index d8662645ca8..42bcce0b4e6 100644 --- a/dbms/src/DataStreams/VerticalRowOutputStream.cpp +++ b/dbms/src/DataStreams/VerticalRowOutputStream.cpp @@ -20,7 +20,7 @@ VerticalRowOutputStream::VerticalRowOutputStream(WriteBuffer & ostr_, const Bloc typedef std::vector Widths_t; Widths_t name_widths(columns); size_t max_name_width = 0; - + for (size_t i = 0; i < columns; ++i) { data_types[i] = sample.getByPosition(i).type; @@ -41,7 +41,7 @@ void VerticalRowOutputStream::writeField(const Field & field) writeEscapedString(names[field_number], ostr); writeCString(": ", ostr); writeString(pads[field_number], ostr); - + data_types[field_number]->serializeTextEscaped(field, ostr); writeChar('\n', ostr); diff --git a/dbms/src/Functions/FunctionsMiscellaneous.cpp b/dbms/src/Functions/FunctionsMiscellaneous.cpp index a5a3901da9c..945bbcde7ef 100644 --- a/dbms/src/Functions/FunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/FunctionsMiscellaneous.cpp @@ -1,6 +1,5 @@ #include -#include #include #include #include @@ -310,70 +309,6 @@ void FunctionVisibleWidth::execute(Block & block, const ColumnNumbers & argument } -/// TODO: Убрать copy-paste из FunctionsConditional.h -template -struct DataTypeFromFieldTypeOrError -{ - static DataTypePtr getDataType() - { - return new typename DataTypeFromFieldType::Type; - } -}; - -template <> -struct DataTypeFromFieldTypeOrError -{ - static DataTypePtr getDataType() - { - return nullptr; - } -}; - -template -DataTypePtr getSmallestCommonNumericTypeImpl() -{ - using ResultType = typename NumberTraits::ResultOfIf::Type; - auto type_res = DataTypeFromFieldTypeOrError::getDataType(); - if (!type_res) - throw Exception("Types " + TypeName::get() + " and " + TypeName::get() - + " are not upscalable to a common type without loss of precision", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - return type_res; -} - -template -DataTypePtr getSmallestCommonNumericTypeLeft(const IDataType & t2) -{ - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); - - throw Exception("Logical error: not a numeric type passed to function getSmallestCommonNumericType", ErrorCodes::LOGICAL_ERROR); -} - -DataTypePtr getSmallestCommonNumericType(const IDataType & t1, const IDataType & t2) -{ - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); - - throw Exception("Logical error: not a numeric type passed to function getSmallestCommonNumericType", ErrorCodes::LOGICAL_ERROR); -} - } @@ -403,8 +338,6 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); - - factory.registerFunction(); } } diff --git a/dbms/src/Functions/FunctionsTransform.cpp b/dbms/src/Functions/FunctionsTransform.cpp new file mode 100644 index 00000000000..dc7af424842 --- /dev/null +++ b/dbms/src/Functions/FunctionsTransform.cpp @@ -0,0 +1,84 @@ +#include +#include +#include + + +namespace DB +{ + +/// TODO: Убрать copy-paste из FunctionsConditional.h +template +struct DataTypeFromFieldTypeOrError +{ + static DataTypePtr getDataType() + { + return new typename DataTypeFromFieldType::Type; + } +}; + +template <> +struct DataTypeFromFieldTypeOrError +{ + static DataTypePtr getDataType() + { + return nullptr; + } +}; + +template +DataTypePtr getSmallestCommonNumericTypeImpl() +{ + using ResultType = typename NumberTraits::ResultOfIf::Type; + auto type_res = DataTypeFromFieldTypeOrError::getDataType(); + if (!type_res) + throw Exception("Types " + TypeName::get() + " and " + TypeName::get() + + " are not upscalable to a common type without loss of precision", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return type_res; +} + +template +DataTypePtr getSmallestCommonNumericTypeLeft(const IDataType & t2) +{ + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + if (typeid_cast(&t2)) return getSmallestCommonNumericTypeImpl(); + + throw Exception("Logical error: not a numeric type passed to function getSmallestCommonNumericType", ErrorCodes::LOGICAL_ERROR); +} + +DataTypePtr getSmallestCommonNumericType(const IDataType & t1, const IDataType & t2) +{ + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + if (typeid_cast(&t1)) return getSmallestCommonNumericTypeLeft(t2); + + throw Exception("Logical error: not a numeric type passed to function getSmallestCommonNumericType", ErrorCodes::LOGICAL_ERROR); +} + +} + + +namespace DB +{ + +void registerFunctionsMiscellaneous(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} From 1cf67bdedec48ffd6babe4673bf62b8179b71dd0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Jun 2015 06:32:37 +0300 Subject: [PATCH 061/109] dbms: addition to prev. revision [#METR-15987]. --- dbms/src/Functions/FunctionFactory.cpp | 2 ++ dbms/src/Functions/FunctionsTransform.cpp | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/dbms/src/Functions/FunctionFactory.cpp b/dbms/src/Functions/FunctionFactory.cpp index 8cecb81377d..d58b18385fe 100644 --- a/dbms/src/Functions/FunctionFactory.cpp +++ b/dbms/src/Functions/FunctionFactory.cpp @@ -29,6 +29,7 @@ void registerFunctionsStringSearch(FunctionFactory &); void registerFunctionsURL(FunctionFactory &); void registerFunctionsVisitParam(FunctionFactory &); void registerFunctionsMath(FunctionFactory &); +void registerFunctionsTransform(FunctionFactory &); FunctionFactory::FunctionFactory() @@ -55,6 +56,7 @@ FunctionFactory::FunctionFactory() registerFunctionsURL(*this); registerFunctionsVisitParam(*this); registerFunctionsMath(*this); + registerFunctionsTransform(*this); } diff --git a/dbms/src/Functions/FunctionsTransform.cpp b/dbms/src/Functions/FunctionsTransform.cpp index dc7af424842..aac7f3527eb 100644 --- a/dbms/src/Functions/FunctionsTransform.cpp +++ b/dbms/src/Functions/FunctionsTransform.cpp @@ -76,7 +76,7 @@ DataTypePtr getSmallestCommonNumericType(const IDataType & t1, const IDataType & namespace DB { -void registerFunctionsMiscellaneous(FunctionFactory & factory) +void registerFunctionsTransform(FunctionFactory & factory) { factory.registerFunction(); } From 8d09c7f6182772e3d9172c9549111163a2238aad Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Jun 2015 06:55:29 +0300 Subject: [PATCH 062/109] dbms: additions [#METR-15987]. --- .../include/DB/Functions/FunctionsTransform.h | 42 ++++++---- ...0165_transform_non_const_default.reference | 84 +++++++++++++++++++ .../00165_transform_non_const_default.sql | 12 +++ 3 files changed, 120 insertions(+), 18 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00165_transform_non_const_default.reference create mode 100644 dbms/tests/queries/0_stateless/00165_transform_non_const_default.sql diff --git a/dbms/include/DB/Functions/FunctionsTransform.h b/dbms/include/DB/Functions/FunctionsTransform.h index b1b3a2ce79a..87419d67bf7 100644 --- a/dbms/include/DB/Functions/FunctionsTransform.h +++ b/dbms/include/DB/Functions/FunctionsTransform.h @@ -729,27 +729,33 @@ private: if (arguments.size() == 4) { - const IColumnConst * default_col = dynamic_cast(&*block.getByPosition(arguments[3]).column); + const IColumn * default_col = block.getByPosition(arguments[3]).column.get(); + const IColumnConst * const_default_col = dynamic_cast(default_col); - if (default_col) + if (const_default_col) + const_default_value = (*const_default_col)[0]; + + /// Нужно ли преобразовать элементы to и default_value к наименьшему общему типу, который является Float64? + bool default_col_is_float = + typeid_cast(default_col) + || typeid_cast(default_col) + || typeid_cast(default_col) + || typeid_cast(default_col); + + bool to_is_float = to[0].getType() == Field::Types::Float64; + + if (default_col_is_float && !to_is_float) { - const_default_value = (*default_col)[0]; - - /// Нужно ли преобразовать элементы to и default_value к наименьшему общему типу, который является Float64? - if (const_default_value.getType() == Field::Types::Float64 && to[0].getType() != Field::Types::Float64) - { - converted_to.resize(to.size()); - for (size_t i = 0, size = to.size(); i < size; ++i) - converted_to[i] = apply_visitor(FieldVisitorConvertToNumber(), to[i]); - used_to = &converted_to; - } - else if (const_default_value.getType() != Field::Types::Float64 && to[0].getType() == Field::Types::Float64) - { - const_default_value = apply_visitor(FieldVisitorConvertToNumber(), const_default_value); - } + converted_to.resize(to.size()); + for (size_t i = 0, size = to.size(); i < size; ++i) + converted_to[i] = apply_visitor(FieldVisitorConvertToNumber(), to[i]); + used_to = &converted_to; + } + else if (!default_col_is_float && to_is_float) + { + if (const_default_col) + const_default_value = apply_visitor(FieldVisitorConvertToNumber(), const_default_value); } - - /// TODO Преобразование для to в случае неконстантного default } /// Замечание: не делается проверка дубликатов в массиве from. diff --git a/dbms/tests/queries/0_stateless/00165_transform_non_const_default.reference b/dbms/tests/queries/0_stateless/00165_transform_non_const_default.reference new file mode 100644 index 00000000000..d66471d9741 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00165_transform_non_const_default.reference @@ -0,0 +1,84 @@ +9999 +9999 +9999 +111 +9999 +222 +9999 +333 +9999 +9999 + + + +hello + +world + +abc + + + + + +hello + +world + +abc + + +- +- +- +hello +- +world +- +abc +- +- +0 +0 +0 +111 +0 +222 +0 +333 +0 +0 +-1 +-1 +-1 +111 +-1 +222 +-1 +333 +-1 +-1 +-1.1 +-1.1 +-1.1 +111 +-1.1 +222 +-1.1 +333 +-1.1 +-1.1 +1 +1 +1 +111 +1 +222.2 +1 +333 +1 +1 +Остальные +Яндекс +Google +Остальные diff --git a/dbms/tests/queries/0_stateless/00165_transform_non_const_default.sql b/dbms/tests/queries/0_stateless/00165_transform_non_const_default.sql new file mode 100644 index 00000000000..f68327f7700 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00165_transform_non_const_default.sql @@ -0,0 +1,12 @@ +SELECT transform(number, [3, 5, 7], [111, 222, 333], materialize(9999)) FROM system.numbers LIMIT 10; +SELECT transform(number, [3, 5, 7], ['hello', 'world', 'abc'], materialize('')) FROM system.numbers LIMIT 10; +SELECT transform(toString(number), ['3', '5', '7'], ['hello', 'world', 'abc'], materialize('')) FROM system.numbers LIMIT 10; +SELECT transform(toString(number), ['3', '5', '7'], ['hello', 'world', 'abc'], materialize('-')) FROM system.numbers LIMIT 10; +SELECT transform(toString(number), ['3', '5', '7'], [111, 222, 333], materialize(0)) FROM system.numbers LIMIT 10; +SELECT transform(toString(number), ['3', '5', '7'], [111, 222, 333], materialize(-1)) FROM system.numbers LIMIT 10; +SELECT transform(toString(number), ['3', '5', '7'], [111, 222, 333], materialize(-1.1)) FROM system.numbers LIMIT 10; +SELECT transform(toString(number), ['3', '5', '7'], [111, 222.2, 333], materialize(1)) FROM system.numbers LIMIT 10; +SELECT transform(1, [2, 3], ['Яндекс', 'Google'], materialize('Остальные')) AS title; +SELECT transform(2, [2, 3], ['Яндекс', 'Google'], materialize('Остальные')) AS title; +SELECT transform(3, [2, 3], ['Яндекс', 'Google'], materialize('Остальные')) AS title; +SELECT transform(4, [2, 3], ['Яндекс', 'Google'], materialize('Остальные')) AS title; From 027342ec5a1c4fad55cc0a01b9a84de6a6d58e15 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 3 Jun 2015 06:56:40 +0300 Subject: [PATCH 063/109] Merge --- dbms/src/Storages/StorageMerge.cpp | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/dbms/src/Storages/StorageMerge.cpp b/dbms/src/Storages/StorageMerge.cpp index ff6e41aa7d3..61397f7c3e3 100644 --- a/dbms/src/Storages/StorageMerge.cpp +++ b/dbms/src/Storages/StorageMerge.cpp @@ -130,6 +130,12 @@ BlockInputStreams StorageMerge::read( std::multiset values = VirtualColumnUtils::extractSingleValueFromBlock(virtual_columns_block, "_table"); + /** На всякий случай отключаем оптимизацию "перенос в PREWHERE", + * так как нет уверенности, что она работает, когда одна из таблиц MergeTree, а другая - нет. + */ + Settings modified_settings = settings; + modified_settings.optimize_move_to_prewhere = false; + for (size_t i = 0, size = selected_tables.size(); i < size; ++i) { StoragePtr table = selected_tables[i]; @@ -150,7 +156,7 @@ BlockInputStreams StorageMerge::read( real_column_names, modified_query_ast, context, - settings, + modified_settings, tmp_processed_stage, max_block_size, size > threads ? 1 : (threads / size)); From ac80e39c1e80a2199a5be2f8ac0559e80f017a55 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Wed, 3 Jun 2015 14:28:16 +0300 Subject: [PATCH 064/109] dbms: Server: Deleted forgotten garbage from header file. [#METR-16546] --- dbms/include/DB/Parsers/ASTSelectQuery.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/dbms/include/DB/Parsers/ASTSelectQuery.h b/dbms/include/DB/Parsers/ASTSelectQuery.h index fd7d6d71d48..c40fcd52e22 100644 --- a/dbms/include/DB/Parsers/ASTSelectQuery.h +++ b/dbms/include/DB/Parsers/ASTSelectQuery.h @@ -32,9 +32,6 @@ public: /// Переписывает select_expression_list, чтобы вернуть только необходимые столбцы в правильном порядке. void rewriteSelectExpressionList(const Names & column_names); - /// Переписывает select_expression_list, чтобы вернуть только необходимые столбцы в правильном порядке. - void rewriteSelectExpressionList3(const Names & column_names); - ASTPtr clone() const override; public: From 59e894426a3f5ed2a24e2f32bd9a5a5e42e3a680 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Wed, 3 Jun 2015 16:11:59 +0300 Subject: [PATCH 065/109] dbms: Server: Added version function. [#METR-16386] --- .../DB/Functions/FunctionsMiscellaneous.h | 34 +++++++++++++++++-- dbms/src/Functions/FunctionsMiscellaneous.cpp | 2 ++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsMiscellaneous.h b/dbms/include/DB/Functions/FunctionsMiscellaneous.h index 3682a806ac9..bba76d40c86 100644 --- a/dbms/include/DB/Functions/FunctionsMiscellaneous.h +++ b/dbms/include/DB/Functions/FunctionsMiscellaneous.h @@ -1,9 +1,9 @@ #pragma once #include +#include -#include - +#include #include #include #include @@ -25,6 +25,7 @@ #include #include +#include namespace DB { @@ -887,5 +888,34 @@ using FunctionIsFinite = FunctionNumericPredicate; using FunctionIsInfinite = FunctionNumericPredicate; using FunctionIsNaN = FunctionNumericPredicate; +class FunctionVersion : public IFunction +{ +public: + static constexpr auto name = "version"; + static IFunction * create(const Context & context) { return new FunctionVersion; } + + String getName() const override { return name; } + + DataTypePtr getReturnType(const DataTypes & arguments) const override + { + if (!arguments.empty()) + throw Exception("Function " + getName() + " must be called without arguments", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + return new DataTypeString; + } + + void execute(Block & block, const ColumnNumbers & arguments, size_t result) override + { + static const std::string version = getVersion(); + block.getByPosition(result).column = new ColumnConstString(version.length(), version); + } + +private: + std::string getVersion() const + { + std::ostringstream os; + os << DBMS_VERSION_MAJOR << "." << DBMS_VERSION_MINOR << "." << Revision::get(); + return os.str(); + } +}; } diff --git a/dbms/src/Functions/FunctionsMiscellaneous.cpp b/dbms/src/Functions/FunctionsMiscellaneous.cpp index 945bbcde7ef..b0cbe7b283a 100644 --- a/dbms/src/Functions/FunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/FunctionsMiscellaneous.cpp @@ -338,6 +338,8 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + + factory.registerFunction(); } } From de8ad3116167fd72e2c80f9a75e6a70fdebf4733 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Wed, 3 Jun 2015 16:20:19 +0300 Subject: [PATCH 066/109] dbms: Server: Added comment. [#METR-16386] --- dbms/include/DB/Functions/FunctionsMiscellaneous.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbms/include/DB/Functions/FunctionsMiscellaneous.h b/dbms/include/DB/Functions/FunctionsMiscellaneous.h index bba76d40c86..77b55120b44 100644 --- a/dbms/include/DB/Functions/FunctionsMiscellaneous.h +++ b/dbms/include/DB/Functions/FunctionsMiscellaneous.h @@ -56,6 +56,8 @@ namespace DB * sleep(n) - спит n секунд каждый блок. * * bar(x, min, max, width) - рисует полосу из количества символов, пропорционального (x - min) и равного width при x == max. + * + * version() - возвращает текущую версию сервера в строке. */ From ce1e437c093e03f7a3a78ca28e31dc6ba43fd119 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Wed, 3 Jun 2015 17:27:03 +0300 Subject: [PATCH 067/109] dbms: Server: Added function toStartOfFiveMinute. [#METR-14495] --- dbms/include/DB/Functions/FunctionsDateTime.h | 15 +++++++++++++-- dbms/src/Functions/FunctionsDateTime.cpp | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsDateTime.h b/dbms/include/DB/Functions/FunctionsDateTime.h index e35c1051f7d..6c338122250 100644 --- a/dbms/include/DB/Functions/FunctionsDateTime.h +++ b/dbms/include/DB/Functions/FunctionsDateTime.h @@ -17,8 +17,8 @@ namespace DB /** Функции работы с датой и временем. * * toYear, toMonth, toDayOfMonth, toDayOfWeek, toHour, toMinute, toSecond, - * toMonday, toStartOfMonth, toStartOfYear, toStartOfMinute, toStartOfHour - * toTime, + * toMonday, toStartOfMonth, toStartOfYear, toStartOfMinute, toStartOfFiveMinute + * toStartOfHour, toTime, * now * TODO: makeDate, makeDateTime * @@ -141,6 +141,15 @@ struct ToStartOfMinuteImpl } }; +struct ToStartOfFiveMinuteImpl +{ + static inline UInt32 execute(UInt32 t, DateLUT & date_lut) { return date_lut.toStartOfFiveMinuteInaccurate(t); } + static inline UInt32 execute(UInt16 d, DateLUT & date_lut) + { + throw Exception("Illegal type Date of argument for function toStartOfFiveMinute", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + } +}; + struct ToStartOfHourImpl { static inline UInt32 execute(UInt32 t, DateLUT & date_lut) { return date_lut.toStartOfHourInaccurate(t); } @@ -597,6 +606,7 @@ struct NameToStartOfMonth { static constexpr auto name = "toStartOfMonth"; }; struct NameToStartOfQuarter { static constexpr auto name = "toStartOfQuarter"; }; struct NameToStartOfYear { static constexpr auto name = "toStartOfYear"; }; struct NameToStartOfMinute { static constexpr auto name = "toStartOfMinute"; }; +struct NameToStartOfFiveMinute { static constexpr auto name = "toStartOfFiveMinute"; }; struct NameToStartOfHour { static constexpr auto name = "toStartOfHour"; }; struct NameToTime { static constexpr auto name = "toTime"; }; struct NameToRelativeYearNum { static constexpr auto name = "toRelativeYearNum"; }; @@ -620,6 +630,7 @@ typedef FunctionDateOrDateTimeToSomething FunctionToStartOfQuarter; typedef FunctionDateOrDateTimeToSomething FunctionToStartOfYear; typedef FunctionDateOrDateTimeToSomething FunctionToStartOfMinute; +typedef FunctionDateOrDateTimeToSomething FunctionToStartOfFiveMinute; typedef FunctionDateOrDateTimeToSomething FunctionToStartOfHour; typedef FunctionDateOrDateTimeToSomething FunctionToTime; diff --git a/dbms/src/Functions/FunctionsDateTime.cpp b/dbms/src/Functions/FunctionsDateTime.cpp index 73503b3e495..dafd7381f3f 100644 --- a/dbms/src/Functions/FunctionsDateTime.cpp +++ b/dbms/src/Functions/FunctionsDateTime.cpp @@ -18,6 +18,7 @@ void registerFunctionsDateTime(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); + factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); From 1037433f3d9334a26142a6d3f7c08a802d68b092 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Wed, 3 Jun 2015 19:08:32 +0300 Subject: [PATCH 068/109] dbms: finally fix out-of-bounds access in SSE lower/upper UTF8[#METR-14764] --- dbms/include/DB/Functions/FunctionsString.h | 31 ++++++++------------- 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index a3c530217a9..52f9f73f98d 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -342,7 +342,6 @@ private: const auto v_zero = _mm_setzero_si128(); const auto v_not_case_lower_bound = _mm_set1_epi8(not_case_lower_bound - 1); const auto v_not_case_upper_bound = _mm_set1_epi8(not_case_upper_bound + 1); -// const auto v_not_case_range = _mm_set1_epi16((not_case_upper_bound << 8) | not_case_lower_bound); const auto v_flip_case_mask = _mm_set1_epi8(flip_case_mask); while (src < src_end_sse) @@ -360,10 +359,6 @@ private: _mm_cmplt_epi8(chars, v_not_case_upper_bound)); const auto mask_is_not_case = _mm_movemask_epi8(is_not_case); - /// check for case - // const auto is_case_result = _mm_cmpestra(v_not_case_range, 2, chars, 16, _SIDD_UBYTE_OPS | _SIDD_CMP_RANGES); - // if (is_case_result == 0) - /// everything in correct case ASCII if (mask_is_not_case == 0) _mm_storeu_si128(reinterpret_cast<__m128i *>(dst), chars); @@ -385,9 +380,9 @@ private: else { /// UTF-8 - const auto end = src + bytes_sse; + const auto expected_end = src + bytes_sse; - while (src < end) + while (src < expected_end) { if (src[0] <= ascii_upper_bound) { @@ -434,22 +429,20 @@ private: // res_pos += 3; // } else - { if (const auto chars = utf8.convert(to_case(utf8.convert(src)), dst, src_end - src)) - { - src += chars; - dst += chars; - } + src += chars, dst += chars; else - { - ++src; - ++dst; - } - } + ++src, ++dst; } - const auto diff = src - end; - src_end_sse += diff; + const auto diff = src - expected_end; + if (diff != 0) + { + if (src_end_sse + diff < src_end) + src_end_sse += diff; + else + src_end_sse -= bytes_sse - diff; + } } } From b86aa4785cd283845c4c07a4ce110afb10386aac Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Wed, 3 Jun 2015 20:01:30 +0300 Subject: [PATCH 069/109] dbms: Server: Avoid negative zeroes in all of the remaining cases. [#METR-15210] --- dbms/include/DB/Functions/FunctionsRound.h | 356 ++++++++++-------- .../00161_rounding_functions.reference | 3 + .../0_stateless/00161_rounding_functions.sql | 5 + 3 files changed, 207 insertions(+), 157 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsRound.h b/dbms/include/DB/Functions/FunctionsRound.h index 2aeb88cf908..7706c86519b 100644 --- a/dbms/include/DB/Functions/FunctionsRound.h +++ b/dbms/include/DB/Functions/FunctionsRound.h @@ -257,92 +257,25 @@ namespace DB }; template - struct BaseFloatRoundingComputation; + class BaseFloatRoundingComputation; template<> - struct BaseFloatRoundingComputation + class BaseFloatRoundingComputation { + public: using Scale = __m128; static const size_t data_count = 4; - }; - template<> - struct BaseFloatRoundingComputation - { - using Scale = __m128d; - static const size_t data_count = 2; - }; - - /** Реализация низкоуровневых функций округления для значений с плавающей точкой. - */ - template - struct FloatRoundingComputation : public BaseFloatRoundingComputation - { - }; - - template - struct FloatRoundingComputation - : public BaseFloatRoundingComputation - { - static inline void prepare(size_t scale, Scale & mm_scale) + protected: + /// Предотвратить появление отрицательных нолей определённых в стандарте IEEE-754. + static inline void normalize(__m128 & val, const __m128 & mask) { - Float32 fscale = static_cast(scale); - mm_scale = _mm_load1_ps(&fscale); - } - - static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) - { - __m128 val = _mm_loadu_ps(in); - val = _mm_mul_ps(val, scale); - val = _mm_round_ps(val, rounding_mode); - val = _mm_div_ps(val, scale); - _mm_storeu_ps(out, val); - } - }; - - template - struct FloatRoundingComputation - : public BaseFloatRoundingComputation - { - static inline void prepare(size_t scale, Scale & mm_scale) - { - Float32 fscale = static_cast(scale); - mm_scale = _mm_load1_ps(&fscale); - } - - static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) - { - __m128 val = _mm_loadu_ps(in); - - /// Превратить отрицательные значения в положительные. - __m128 factor = _mm_cmpge_ps(val, getZero()); - factor = _mm_min_ps(factor, getTwo()); - factor = _mm_sub_ps(factor, getOne()); - val = _mm_mul_ps(val, factor); - - /// Алгоритм округления. - val = _mm_div_ps(val, scale); - __m128 res = _mm_cmpge_ps(val, getOneTenth()); - val = _mm_round_ps(val, rounding_mode); - val = _mm_mul_ps(val, scale); - val = _mm_and_ps(val, res); - - /// Предотвратить появление отрицательных нолей определённых в стандарте IEEE-754. - __m128 check = _mm_cmpeq_ps(val, getZero()); - check = _mm_min_ps(check, getOne()); - factor = _mm_add_ps(factor, check); - - /// Вернуть настоящие знаки всех значений. - val = _mm_mul_ps(val, factor); - - _mm_storeu_ps(out, val); - } - - private: - static inline const __m128 & getOneTenth() - { - static const __m128 one_tenth = _mm_set1_ps(0.1); - return one_tenth; + __m128 mask1 = _mm_cmpeq_ps(val, getZero()); + __m128 mask2 = _mm_and_ps(mask, mask1); + mask2 = _mm_cmpeq_ps(mask2, getZero()); + mask2 = _mm_min_ps(mask2, getTwo()); + mask2 = _mm_sub_ps(mask2, getOne()); + val = _mm_mul_ps(val, mask2); } static inline const __m128 & getZero() @@ -364,85 +297,23 @@ namespace DB } }; - template - struct FloatRoundingComputation - : public BaseFloatRoundingComputation + template<> + class BaseFloatRoundingComputation { - static inline void prepare(size_t scale, Scale & mm_scale) + public: + using Scale = __m128d; + static const size_t data_count = 2; + + protected: + /// Предотвратить появление отрицательных нолей определённых в стандарте IEEE-754. + static inline void normalize(__m128d & val, const __m128d & mask) { - } - - static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) - { - __m128 val = _mm_loadu_ps(in); - val = _mm_round_ps(val, rounding_mode); - _mm_storeu_ps(out, val); - } - }; - - template - struct FloatRoundingComputation - : public BaseFloatRoundingComputation - { - static inline void prepare(size_t scale, Scale & mm_scale) - { - Float64 fscale = static_cast(scale); - mm_scale = _mm_load1_pd(&fscale); - } - - static inline void compute(const Float64 * __restrict in, const Scale & scale, Float64 * __restrict out) - { - __m128d val = _mm_loadu_pd(in); - val = _mm_mul_pd(val, scale); - val = _mm_round_pd(val, rounding_mode); - val = _mm_div_pd(val, scale); - _mm_storeu_pd(out, val); - } - }; - - template - struct FloatRoundingComputation - : public BaseFloatRoundingComputation - { - static inline void prepare(size_t scale, Scale & mm_scale) - { - Float64 fscale = static_cast(scale); - mm_scale = _mm_load1_pd(&fscale); - } - - static inline void compute(const Float64 * __restrict in, const Scale & scale, Float64 * __restrict out) - { - __m128d val = _mm_loadu_pd(in); - - /// Превратить отрицательные значения в положительные. - __m128d factor = _mm_cmpge_pd(val, getZero()); - factor = _mm_min_pd(factor, getTwo()); - factor = _mm_sub_pd(factor, getOne()); - val = _mm_mul_pd(val, factor); - - /// Алгоритм округления. - val = _mm_div_pd(val, scale); - __m128d res = _mm_cmpge_pd(val, getOneTenth()); - val = _mm_round_pd(val, rounding_mode); - val = _mm_mul_pd(val, scale); - val = _mm_and_pd(val, res); - - /// Предотвратить появление отрицательных нолей определённых в стандарте IEEE-754. - __m128d check = _mm_cmpeq_pd(val, getZero()); - check = _mm_min_pd(check, getOne()); - factor = _mm_add_pd(factor, check); - - /// Вернуть настоящие знаки всех значений. - val = _mm_mul_pd(val, factor); - - _mm_storeu_pd(out, val); - } - - private: - static inline const __m128d & getOneTenth() - { - static const __m128d one_tenth = _mm_set1_pd(0.1); - return one_tenth; + __m128d mask1 = _mm_cmpeq_pd(val, getZero()); + __m128d mask2 = _mm_and_pd(mask, mask1); + mask2 = _mm_cmpeq_pd(mask2, getZero()); + mask2 = _mm_min_pd(mask2, getTwo()); + mask2 = _mm_sub_pd(mask2, getOne()); + val = _mm_mul_pd(val, mask2); } static inline const __m128d & getZero() @@ -464,10 +335,177 @@ namespace DB } }; + /** Реализация низкоуровневых функций округления для значений с плавающей точкой. + */ + template + class FloatRoundingComputation; + template - struct FloatRoundingComputation + class FloatRoundingComputation + : public BaseFloatRoundingComputation + { + public: + static inline void prepare(size_t scale, Scale & mm_scale) + { + Float32 fscale = static_cast(scale); + mm_scale = _mm_load1_ps(&fscale); + } + + static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) + { + __m128 val = _mm_loadu_ps(in); + __m128 mask = _mm_cmplt_ps(val, getZero()); + + /// Алгоритм округления. + val = _mm_mul_ps(val, scale); + val = _mm_round_ps(val, rounding_mode); + val = _mm_div_ps(val, scale); + + normalize(val, mask); + _mm_storeu_ps(out, val); + } + }; + + template + class FloatRoundingComputation + : public BaseFloatRoundingComputation + { + public: + static inline void prepare(size_t scale, Scale & mm_scale) + { + Float32 fscale = static_cast(scale); + mm_scale = _mm_load1_ps(&fscale); + } + + static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) + { + __m128 val = _mm_loadu_ps(in); + __m128 mask = _mm_cmplt_ps(val, getZero()); + + /// Превратить отрицательные значения в положительные. + __m128 factor = _mm_cmpge_ps(val, getZero()); + factor = _mm_min_ps(factor, getTwo()); + factor = _mm_sub_ps(factor, getOne()); + val = _mm_mul_ps(val, factor); + + /// Алгоритм округления. + val = _mm_div_ps(val, scale); + __m128 res = _mm_cmpge_ps(val, getOneTenth()); + val = _mm_round_ps(val, rounding_mode); + val = _mm_mul_ps(val, scale); + val = _mm_and_ps(val, res); + + /// Вернуть настоящие знаки всех значений. + val = _mm_mul_ps(val, factor); + + normalize(val, mask); + _mm_storeu_ps(out, val); + } + + private: + static inline const __m128 & getOneTenth() + { + static const __m128 one_tenth = _mm_set1_ps(0.1); + return one_tenth; + } + }; + + template + class FloatRoundingComputation + : public BaseFloatRoundingComputation + { + public: + static inline void prepare(size_t scale, Scale & mm_scale) + { + } + + static inline void compute(const Float32 * __restrict in, const Scale & scale, Float32 * __restrict out) + { + __m128 val = _mm_loadu_ps(in); + __m128 mask = _mm_cmplt_ps(val, getZero()); + + val = _mm_round_ps(val, rounding_mode); + + normalize(val, mask); + _mm_storeu_ps(out, val); + } + }; + + template + class FloatRoundingComputation : public BaseFloatRoundingComputation { + public: + static inline void prepare(size_t scale, Scale & mm_scale) + { + Float64 fscale = static_cast(scale); + mm_scale = _mm_load1_pd(&fscale); + } + + static inline void compute(const Float64 * __restrict in, const Scale & scale, Float64 * __restrict out) + { + __m128d val = _mm_loadu_pd(in); + __m128d mask = _mm_cmplt_pd(val, getZero()); + + /// Алгоритм округления. + val = _mm_mul_pd(val, scale); + val = _mm_round_pd(val, rounding_mode); + val = _mm_div_pd(val, scale); + + normalize(val, mask); + _mm_storeu_pd(out, val); + } + }; + + template + class FloatRoundingComputation + : public BaseFloatRoundingComputation + { + public: + static inline void prepare(size_t scale, Scale & mm_scale) + { + Float64 fscale = static_cast(scale); + mm_scale = _mm_load1_pd(&fscale); + } + + static inline void compute(const Float64 * __restrict in, const Scale & scale, Float64 * __restrict out) + { + __m128d val = _mm_loadu_pd(in); + __m128d mask = _mm_cmplt_pd(val, getZero()); + + /// Превратить отрицательные значения в положительные. + __m128d factor = _mm_cmpge_pd(val, getZero()); + factor = _mm_min_pd(factor, getTwo()); + factor = _mm_sub_pd(factor, getOne()); + val = _mm_mul_pd(val, factor); + + /// Алгоритм округления. + val = _mm_div_pd(val, scale); + __m128d res = _mm_cmpge_pd(val, getOneTenth()); + val = _mm_round_pd(val, rounding_mode); + val = _mm_mul_pd(val, scale); + val = _mm_and_pd(val, res); + + /// Вернуть настоящие знаки всех значений. + val = _mm_mul_pd(val, factor); + + normalize(val, mask); + _mm_storeu_pd(out, val); + } + + private: + static inline const __m128d & getOneTenth() + { + static const __m128d one_tenth = _mm_set1_pd(0.1); + return one_tenth; + } + }; + + template + class FloatRoundingComputation + : public BaseFloatRoundingComputation + { + public: static inline void prepare(size_t scale, Scale & mm_scale) { } @@ -475,7 +513,11 @@ namespace DB static inline void compute(const Float64 * __restrict in, const Scale & scale, Float64 * __restrict out) { __m128d val = _mm_loadu_pd(in); + __m128d mask = _mm_cmplt_pd(val, getZero()); + val = _mm_round_pd(val, rounding_mode); + + normalize(val, mask); _mm_storeu_pd(out, val); } }; diff --git a/dbms/tests/queries/0_stateless/00161_rounding_functions.reference b/dbms/tests/queries/0_stateless/00161_rounding_functions.reference index 9b1be1333d2..009a2080687 100644 --- a/dbms/tests/queries/0_stateless/00161_rounding_functions.reference +++ b/dbms/tests/queries/0_stateless/00161_rounding_functions.reference @@ -852,3 +852,6 @@ 2.738 2.734 2.73 +0 +0 +0 diff --git a/dbms/tests/queries/0_stateless/00161_rounding_functions.sql b/dbms/tests/queries/0_stateless/00161_rounding_functions.sql index 052796b8247..22ab0a01a0c 100644 --- a/dbms/tests/queries/0_stateless/00161_rounding_functions.sql +++ b/dbms/tests/queries/0_stateless/00161_rounding_functions.sql @@ -947,3 +947,8 @@ SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN ran SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(9) AS x); SELECT round(y,3) FROM (SELECT 2.718281828459045 + 1/(1+x*x) AS y ARRAY JOIN range(10) AS x); +/* Negative zeroes. */ + +SELECT round(-0.002); +SELECT round(-0.002, -1); +SELECT round(-0.002, 1); From 05814604de25e77d1f48d5057a598c91d6eef65c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 4 Jun 2015 05:07:30 +0300 Subject: [PATCH 070/109] dbms: fixed error with ATTACH for replicated tables [#MTRSADMIN-1093]. --- .../Storages/StorageReplicatedMergeTree.cpp | 122 +++++++++++++++--- 1 file changed, 103 insertions(+), 19 deletions(-) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index ea7f0072b71..e113fc298d3 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -2511,54 +2511,138 @@ void StorageReplicatedMergeTree::waitForReplicaToProcessLogEntry(const String & { auto zookeeper = getZooKeeper(); - UInt64 log_index = parse(entry.znode_name.substr(entry.znode_name.size() - 10)); - String log_entry_str = entry.toString(); + String entry_str = entry.toString(); + String log_node_name; - LOG_DEBUG(log, "Waiting for " << replica << " to pull " << entry.znode_name << " to queue"); + /** В эту функцию могут передать entry двух видов: + * 1. (более часто) Из директории log - общего лога, откуда реплики копируют записи в свою queue. + * 2. Из директории queue одной из реплик. + * + * Проблема в том, что номера (sequential нод) элементов очереди в log и в queue не совпадают. + * (И в queue не совпадают номера у одного и того же элемента лога для разных реплик.) + * + * Поэтому следует рассматривать эти случаи по-отдельности. + */ - /// Дождемся, пока запись попадет в очередь реплики. - while (true) + /** Первое - нужно дождаться, пока реплика возьмёт к себе в queue элемент очереди из log, + * если она ещё этого не сделала (см. функцию pullLogsToQueue). + * + * Для этого проверяем её узел log_pointer - максимальный номер взятого элемента из log плюс единица. + */ + + if (0 == entry.znode_name.compare(0, strlen("log-"), "log-")) { - zkutil::EventPtr event = new Poco::Event; + /** В этом случае просто берём номер из имени ноды log-xxxxxxxxxx. + */ - String pointer = zookeeper->get(zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); - if (!pointer.empty() && parse(pointer) > log_index) - break; + UInt64 log_index = parse(entry.znode_name.substr(entry.znode_name.size() - 10)); + log_node_name = entry.znode_name; - event->wait(); + LOG_DEBUG(log, "Waiting for " << replica << " to pull " << log_node_name << " to queue"); + + /// Дождемся, пока запись попадет в очередь реплики. + while (true) + { + zkutil::EventPtr event = new Poco::Event; + + String log_pointer = zookeeper->get(zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + if (!log_pointer.empty() && parse(log_pointer) > log_index) + break; + + event->wait(); + } } + else if (0 == entry.znode_name.compare(0, strlen("queue-"), "queue-")) + { + /** В этом случае номер log-ноды неизвестен. Нужно просмотреть все от log_pointer до конца, + * ища ноду с таким же содержимым. И если мы её не найдём - значит реплика уже взяла эту запись в свою queue. + */ - LOG_DEBUG(log, "Looking for " << entry.znode_name << " in " << replica << " queue"); + String log_pointer = zookeeper->get(zookeeper_path + "/replicas/" + replica + "/log_pointer"); + + Strings log_entries = zookeeper->getChildren(zookeeper_path + "/log"); + UInt64 log_index = 0; + bool found = false; + + for (const String & log_entry_name : log_entries) + { + log_index = parse(log_entry_name.substr(log_entry_name.size() - 10)); + + if (!log_pointer.empty() && log_index < parse(log_pointer)) + continue; + + String log_entry_str; + bool exists = zookeeper->tryGet(zookeeper_path + "/log/" + log_entry_name, log_entry_str); + if (exists && entry_str == log_entry_str) + { + found = true; + log_node_name = log_entry_name; + break; + } + } + + if (found) + { + LOG_DEBUG(log, "Waiting for " << replica << " to pull " << log_node_name << " to queue"); + + /// Дождемся, пока запись попадет в очередь реплики. + while (true) + { + zkutil::EventPtr event = new Poco::Event; + + String log_pointer = zookeeper->get(zookeeper_path + "/replicas/" + replica + "/log_pointer", nullptr, event); + if (!log_pointer.empty() && parse(log_pointer) > log_index) + break; + + event->wait(); + } + } + } + else + throw Exception("Logical error: unexpected name of log node: " + entry.znode_name, ErrorCodes::LOGICAL_ERROR); + + if (!log_node_name.empty()) + LOG_DEBUG(log, "Looking for node corresponding to " << log_node_name << " in " << replica << " queue"); + else + LOG_DEBUG(log, "Looking for corresponding node in " << replica << " queue"); + + /** Второе - найдем соответствующую запись в очереди указанной реплики (replica). + * Её номер может не совпадать ни с log-узлом, ни с queue-узлом у текущей реплики (у нас). + * Поэтому, ищем путём сравнения содержимого. + */ - /// Найдем запись в очереди реплики. Strings queue_entries = zookeeper->getChildren(zookeeper_path + "/replicas/" + replica + "/queue"); - String entry_to_wait_for; + String queue_entry_to_wait_for; for (const String & entry_name : queue_entries) { String queue_entry_str; bool exists = zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/queue/" + entry_name, queue_entry_str); - if (exists && queue_entry_str == log_entry_str) + if (exists && queue_entry_str == entry_str) { - entry_to_wait_for = entry_name; + queue_entry_to_wait_for = entry_name; break; } } /// Пока искали запись, ее уже выполнили и удалили. - if (entry_to_wait_for.empty()) + if (queue_entry_to_wait_for.empty()) + { + LOG_DEBUG(log, "No corresponding node found. Assuming it has been already processed."); return; + } - LOG_DEBUG(log, "Waiting for " << entry_to_wait_for << " to disappear from " << replica << " queue"); + LOG_DEBUG(log, "Waiting for " << queue_entry_to_wait_for << " to disappear from " << replica << " queue"); + + /// Третье - дождемся, пока запись исчезнет из очереди реплики. - /// Дождемся, пока запись исчезнет из очереди реплики. while (true) { zkutil::EventPtr event = new Poco::Event; String unused; /// get вместо exists, чтобы не утек watch, если ноды уже нет. - if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/queue/" + entry_to_wait_for, unused, nullptr, event)) + if (!zookeeper->tryGet(zookeeper_path + "/replicas/" + replica + "/queue/" + queue_entry_to_wait_for, unused, nullptr, event)) break; event->wait(); From 9f15e1a483d978f467248a662f6c8919b033e2c3 Mon Sep 17 00:00:00 2001 From: Alexey Arno Date: Thu, 4 Jun 2015 13:28:31 +0300 Subject: [PATCH 071/109] dbms: Server: Small cosmetic change for memset/memcpy/memmove functions. Does not change functionality. [#METR-15090] --- dbms/src/IO/WriteBufferAIO.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dbms/src/IO/WriteBufferAIO.cpp b/dbms/src/IO/WriteBufferAIO.cpp index 229e21be131..392eec2ada0 100644 --- a/dbms/src/IO/WriteBufferAIO.cpp +++ b/dbms/src/IO/WriteBufferAIO.cpp @@ -320,15 +320,15 @@ void WriteBufferAIO::prepare() buffer_size += region_left_padding; buffer_end = buffer_begin + buffer_size; - ::memmove(buffer_begin + region_left_padding, buffer_begin, buffer_size - region_left_padding); + ::memmove(buffer_begin + region_left_padding, buffer_begin, (buffer_size - region_left_padding) * sizeof(*buffer_begin)); ssize_t read_count = ::pread(fd, memory_page, DEFAULT_AIO_FILE_BLOCK_SIZE, region_aligned_begin); if (read_count < 0) throw Exception("Read error", ErrorCodes::AIO_READ_ERROR); size_t to_copy = std::min(static_cast(read_count), region_left_padding); - ::memcpy(buffer_begin, memory_page, to_copy); - ::memset(buffer_begin + to_copy, 0, region_left_padding - to_copy); + ::memcpy(buffer_begin, memory_page, to_copy * sizeof(*buffer_begin)); + ::memset(buffer_begin + to_copy, 0, (region_left_padding - to_copy) * sizeof(*buffer_begin)); } if (region_right_padding > 0) @@ -342,7 +342,7 @@ void WriteBufferAIO::prepare() off_t offset = DEFAULT_AIO_FILE_BLOCK_SIZE - region_right_padding; if (read_count > offset) { - ::memcpy(buffer_end, memory_page + offset, read_count - offset); + ::memcpy(buffer_end, memory_page + offset, (read_count - offset) * sizeof(*buffer_end)); truncation_begin = buffer_end + (read_count - offset); truncation_count = DEFAULT_AIO_FILE_BLOCK_SIZE - read_count; } @@ -352,7 +352,7 @@ void WriteBufferAIO::prepare() truncation_count = region_right_padding; } - ::memset(truncation_begin, 0, truncation_count); + ::memset(truncation_begin, 0, truncation_count * sizeof(*truncation_begin)); } } } From 5a21f3908b054a0efc90c65a12fbe151c74d90dc Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Thu, 4 Jun 2015 15:16:42 +0300 Subject: [PATCH 072/109] dbms: SSE lower/upperUTF8 faster handling of cyrillic sequences[#METR-14764] --- dbms/include/DB/Functions/FunctionsString.h | 119 +++++++++++++------- 1 file changed, 77 insertions(+), 42 deletions(-) diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index 52f9f73f98d..3025aa46942 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -240,8 +240,6 @@ private: template struct LowerUpperImplVectorized { - template friend class LowerUpperUTF8ImplVectorized; - static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets, ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets) { @@ -303,7 +301,55 @@ private: } }; -template + +/// xor or do nothing +template UInt8 xor_or_identity(const UInt8 c, const int mask) { return c ^ mask; }; +template <> inline UInt8 xor_or_identity(const UInt8 c, const int) { return c; } + +/// It is caller's responsibility to ensure the presence of a valid cyrillic sequence in array +template +inline void UTF8CyrillicToCase(const UInt8 * & src, const UInt8 * const src_end, UInt8 * & dst) +{ + if (src[0] == 0xD0u && (src[1] >= 0x80u && src[1] <= 0x8Fu)) + { + /// ЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏ + *dst++ = xor_or_identity(*src++, 0x1); + *dst++ = xor_or_identity(*src++, 0x10); + } + else if (src[0] == 0xD1u && (src[1] >= 0x90u && src[1] <= 0x9Fu)) + { + /// ѐёђѓєѕіїјљњћќѝўџ + *dst++ = xor_or_identity(*src++, 0x1); + *dst++ = xor_or_identity(*src++, 0x10); + } + else if (src[0] == 0xD0u && (src[1] >= 0x90u && src[1] <= 0x9Fu)) + { + /// А-П + *dst++ = *src++; + *dst++ = xor_or_identity(*src++, 0x20); + } + else if (src[0] == 0xD0u && (src[1] >= 0xB0u && src[1] <= 0xBFu)) + { + /// а-п + *dst++ = *src++; + *dst++ = xor_or_identity(*src++, 0x20); + } + else if (src[0] == 0xD0u && (src[1] >= 0xA0u && src[1] <= 0xAFu)) + { + /// Р-Я + *dst++ = xor_or_identity(*src++, 0x1); + *dst++ = xor_or_identity(*src++, 0x20); + } + else if (src[0] == 0xD1u && (src[1] >= 0x80u && src[1] <= 0x8Fu)) + { + /// р-я + *dst++ = xor_or_identity(*src++, 0x1); + *dst++ = xor_or_identity(*src++, 0x20); + } +}; + +template struct LowerUpperUTF8ImplVectorized { static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets, @@ -391,50 +437,35 @@ private: else *dst++ = *src++; } -// else if (src + 1 < src_end /// кириллица: русский алфавит, а также ЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏ -// && ((src[0] == '\xD0' && (src[1] >= '\x80' && src[1] <= '\xBF')) -// || (src[0] == '\xD1' && (src[1] >= '\x80' && src[1] <= '\x9F')))) -// { -// /** d0 80-8f -> +1 +10 -// * d0 90-9f -> +0 +20 -// * d0 a0-af -> +1 -20 -// */ -// -// if (src[0] == '\xD0' && (src[1] >= '\x80' && src[1] <= '\x8F')) -// { -// ++res_pos[0]; -// res_pos[1] += 0x10; -// } -// else if (src[0] == '\xD0' && (src[1] >= '\x90' && src[1] <= '\x9F')) -// { -// res_pos[1] += 0x20; -// } -// else if (src[0] == '\xD0' && (src[1] >= '\xA0' && src[1] <= '\xAF')) -// { -// ++res_pos[0]; -// res_pos[1] -= 0x20; -// } -// -// pos += 2; -// res_pos += 2; -// } -// else if (pos + 1 < src_end && pos[0] == '\xC2') /// Пунктуация U+0080 - U+00BF, UTF-8: C2 80 - C2 BF -// { -// pos += 2; -// res_pos += 2; -// } -// else if (pos + 2 < src_end && pos[0] == '\xE2') /// Символы U+2000 - U+2FFF, UTF-8: E2 80 80 - E2 BF BF -// { -// pos += 3; -// res_pos += 3; -// } + else if (src + 1 < src_end && + ((src[0] == 0xD0u && (src[1] >= 0x80u && src[1] <= 0xBFu)) || + (src[0] == 0xD1u && (src[1] >= 0x80u && src[1] <= 0x9Fu)))) + { + cyrillic_to_case(src, src_end, dst); + } + else if (src + 1 < src_end && src[0] == 0xC2u) + { + /// Пунктуация U+0080 - U+00BF, UTF-8: C2 80 - C2 BF + *dst++ = *src++; + *dst++ = *src++; + } + else if (src + 2 < src_end && src[0] == 0xE2u) + { + /// Символы U+2000 - U+2FFF, UTF-8: E2 80 80 - E2 BF BF + *dst++ = *src++; + *dst++ = *src++; + *dst++ = *src++; + } else + { if (const auto chars = utf8.convert(to_case(utf8.convert(src)), dst, src_end - src)) src += chars, dst += chars; else ++src, ++dst; + } } + /// adjust src_end_sse by pushing it forward or backward const auto diff = src - expected_end; if (diff != 0) { @@ -1665,8 +1696,12 @@ typedef FunctionStringNumNumToString Func using FunctionSSELower = FunctionStringToString, NameSSELower>; using FunctionSSEUpper = FunctionStringToString, NameSSEUpper>; -using FunctionSSELowerUTF8 = FunctionStringToString, NameSSELowerUTF8>; -using FunctionSSEUpperUTF8 = FunctionStringToString, NameSSEUpperUTF8>; +using FunctionSSELowerUTF8 = FunctionStringToString< + LowerUpperUTF8ImplVectorized<'A', 'Z', Poco::Unicode::toLower, UTF8CyrillicToCase>, + NameSSELowerUTF8>; +using FunctionSSEUpperUTF8 = FunctionStringToString< + LowerUpperUTF8ImplVectorized<'a', 'z', Poco::Unicode::toUpper, UTF8CyrillicToCase>, + NameSSEUpperUTF8>; } From 4899dd47aa297e8c4991612168bbd5af7e7140a0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Jun 2015 19:41:18 +0300 Subject: [PATCH 073/109] dbms: get rid of 'increment.txt': addition [#METR-16629]. --- dbms/src/Storages/StorageMergeTree.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index 66f55601524..f9e9f376fe2 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -43,6 +43,25 @@ StorageMergeTree::StorageMergeTree( data.loadDataParts(false); data.clearOldParts(); increment.set(data.getMaxDataPartIndex()); + + /** Если остался старый (не использующийся сейчас) файл increment.txt, то удалим его. + * Это нужно сделать, чтобы избежать ситуации, когда из-за копирования данных + * от сервера с новой версией (но с оставшимся некорректным и неиспользуемым increment.txt) + * на сервер со старой версией (где increment.txt используется), + * будет скопирован и использован некорректный increment.txt. + * + * Это - защита от очень редкого гипотетического случая. + * Он может достигаться в БК, где довольно медленно обновляют ПО, + * но зато часто делают копирование данных rsync-ом. + */ + { + Poco::File obsolete_increment_txt(full_path + "increment.txt"); + if (obsolete_increment_txt.exists()) + { + LOG_INFO(log, "Removing obsolete file " << full_path << "increment.txt"); + obsolete_increment_txt.remove(); + } + } } StoragePtr StorageMergeTree::create( From 0326ee5a2e2883bb4df42ba3ad6ec5bd5b25722d Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Fri, 5 Jun 2015 20:30:24 +0300 Subject: [PATCH 074/109] Merge --- .../MergeTree/MergeTreeWhereOptimizer.h | 62 +++++++++++++------ 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h b/dbms/include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h index 8ec5657b756..dcdcb28e8b8 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -9,12 +9,14 @@ #include #include #include +#include #include #include #include #include #include + namespace DB { @@ -96,28 +98,33 @@ private: { const auto condition = conditions[idx].get(); + /// linearize sub-conjunctions + if (const auto function = typeid_cast(condition)) + { + if (function->name == and_function_name) + { + for (auto & child : function->arguments->children) + conditions.emplace_back(std::move(child)); + + /// remove the condition corresponding to conjunction + remove_condition_at_index(idx); + + /// continue iterating without increment to ensure the just added conditions are processed + continue; + } + } + + SCOPE_EXIT(++idx); + + if (hasRestrictedFunctions(condition)) + continue; + IdentifierNameSet identifiers{}; collectIdentifiersNoSubqueries(condition, identifiers); /// do not take into consideration the conditions consisting only of primary key columns if (hasNonPrimaryKeyColumns(identifiers) && isSubsetOfTableColumns(identifiers)) { - /// linearize sub-conjunctions - if (const auto function = typeid_cast(condition)) - { - if (function->name == and_function_name) - { - for (auto & child : function->arguments->children) - conditions.emplace_back(std::move(child)); - - /// remove the condition corresponding to conjunction - remove_condition_at_index(idx); - - /// continue iterating without increment to ensure the just added conditions are processed - continue; - } - } - /// calculate size of columns involved in condition const auto cond_columns_size = getIdentifiersColumnSize(identifiers); @@ -129,8 +136,6 @@ private: good_or_viable_condition.second = cond_columns_size; } } - - ++idx; } const auto move_condition_to_prewhere = [&] (const std::size_t idx) { @@ -180,6 +185,10 @@ private: { auto & condition = select.where_expression; + /// do not optimize restricted expressions + if (hasRestrictedFunctions(select.where_expression.get())) + return; + IdentifierNameSet identifiers{}; collectIdentifiersNoSubqueries(condition, identifiers); @@ -300,6 +309,23 @@ private: return true; } + /// we assume all AS aliases have been expanded previously + static bool hasRestrictedFunctions(const IAST * ptr) + { + if (const auto function_ptr = typeid_cast(ptr)) + { + /// disallow arrayJoin expressions to be moved to PREWHERE for now + if ("arrayJoin" == function_ptr->name) + return true; + } + + for (const auto & child : ptr->children) + if (hasRestrictedFunctions(child.get())) + return true; + + return false; + } + string_set_t primary_key_columns{}; string_set_t table_columns{}; Logger * log; From 58465191e85ddd399d5008a1cf946b1f580ed949 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Jun 2015 20:34:57 +0300 Subject: [PATCH 075/109] dbms: fixed error: stuck with too much parts [#METR-16703]. --- dbms/include/DB/Common/ProfileEvents.h | 1 + .../MergeTree/MergeTreeBlockOutputStream.h | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/dbms/include/DB/Common/ProfileEvents.h b/dbms/include/DB/Common/ProfileEvents.h index d664f5fe3a8..10aebbbeef4 100644 --- a/dbms/include/DB/Common/ProfileEvents.h +++ b/dbms/include/DB/Common/ProfileEvents.h @@ -51,6 +51,7 @@ M(DelayedInserts) \ M(RejectedInserts) \ M(DelayedInsertsMilliseconds) \ + M(SynchronousMergeOnInsert) \ \ M(ZooKeeperInit) \ M(ZooKeeperTransactions) \ diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockOutputStream.h b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockOutputStream.h index c2b63669e34..80d3788eafc 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockOutputStream.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockOutputStream.h @@ -15,7 +15,17 @@ public: void writePrefix() override { - storage.data.delayInsertIfNeeded(); + /// Если слишком много кусков - делаем внеочередные мерджи, синхронно, в текущем потоке. + /// Почему 10? - на всякий случай, вместо бесконечного цикла. + for (size_t i = 0; i < 10; ++i) + { + size_t parts_count = storage.data.getMaxPartsCountForMonth(); + if (parts_count <= storage.data.settings.parts_to_delay_insert) + break; + + ProfileEvents::increment(ProfileEvents::SynchronousMergeOnInsert); + storage.merge(0, true); + } } void write(const Block & block) override @@ -26,6 +36,8 @@ public: UInt64 temp_index = storage.increment.get(); MergeTreeData::MutableDataPartPtr part = storage.writer.writeTempPart(current_block, temp_index); storage.data.renameTempPartAndAdd(part, &storage.increment); + + /// Инициируем асинхронный мердж - он будет произведён, если пора делать мердж и если в background_pool-е есть место. storage.merge_task_handle->wake(); } } From b4f60297d23d1e1e7e1a358ade0fcea31ba340b1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Jun 2015 22:33:52 +0300 Subject: [PATCH 076/109] dbms: Client: max_block_sizes: using parameters from Settings as defaults [#METR-2944]. --- dbms/src/Client/Client.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/src/Client/Client.cpp b/dbms/src/Client/Client.cpp index 6039609e75d..103ba626a4c 100644 --- a/dbms/src/Client/Client.cpp +++ b/dbms/src/Client/Client.cpp @@ -263,10 +263,10 @@ private: else format = config().getString("format", is_interactive ? "PrettyCompact" : "TabSeparated"); - format_max_block_size = config().getInt("format_max_block_size", DEFAULT_BLOCK_SIZE); + format_max_block_size = config().getInt("format_max_block_size", context.getSettingsRef().max_block_size); insert_format = "Values"; - insert_format_max_block_size = config().getInt("insert_format_max_block_size", DEFAULT_INSERT_BLOCK_SIZE); + insert_format_max_block_size = config().getInt("insert_format_max_block_size", context.getSettingsRef().max_insert_block_size); connect(); From cac3ce7e271e02ded4e415f82ce7f8a29450ddc4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 5 Jun 2015 23:04:54 +0300 Subject: [PATCH 077/109] dbms: allowed more easily to specify cluster name with hyphens [#METR-16610]. --- dbms/src/Parsers/ExpressionListParsers.cpp | 2 +- dbms/src/Storages/StorageFactory.cpp | 28 +++++++++++++++++++++- 2 files changed, 28 insertions(+), 2 deletions(-) diff --git a/dbms/src/Parsers/ExpressionListParsers.cpp b/dbms/src/Parsers/ExpressionListParsers.cpp index 0fc7ab7fead..0bbcd94b3f0 100644 --- a/dbms/src/Parsers/ExpressionListParsers.cpp +++ b/dbms/src/Parsers/ExpressionListParsers.cpp @@ -112,6 +112,7 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, Pos end, ASTP { bool first = true; ParserWhiteSpaceOrComments ws; + Pos begin = pos; while (1) { @@ -129,7 +130,6 @@ bool ParserLeftAssociativeBinaryOperatorList::parseImpl(Pos & pos, Pos end, ASTP ws.ignore(pos, end); /// пробуем найти какой-нибудь из допустимых операторов - Pos begin = pos; const char ** it; for (it = operators; *it; it += 2) diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 66c4c57f5e9..9251d2e87b9 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -299,7 +299,33 @@ StoragePtr StorageFactory::get( if (args.size() != 3 && args.size() != 4) throw Exception(params_error_message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - String cluster_name = typeid_cast(*args[0]).name; + /** Имя кластера - это имя тега в xml-конфигурации. + * Обычно оно парсится как идентификатор. То есть, оно может содержать подчёркивания, но не может содержать дефисы, + * при условии, что идентификатор не находится в обратных кавычках. + * Но в xml в качестве имени тега более привычно использовать дефисы. + * Такое имя будет парситься как выражение с оператором минус - совсем не то, что нужно. + * Поэтому, рассмотрим такой случай отдельно. + */ + String cluster_name; + + if (const ASTIdentifier * ast_id = typeid_cast(args[0].get())) + { + cluster_name = ast_id->name; + } + else if (const ASTLiteral * ast_lit = typeid_cast(args[0].get())) + { + cluster_name = ast_lit->value.safeGet(); + } + else if (const ASTFunction * ast_func = typeid_cast(args[0].get())) + { + if (!ast_func->range.first || !ast_func->range.second) + throw Exception("Illegal expression instead of cluster name.", ErrorCodes::BAD_ARGUMENTS); + + cluster_name = String(ast_func->range.first, ast_func->range.second); + } + else + throw Exception("Illegal expression instead of cluster name.", ErrorCodes::BAD_ARGUMENTS); + String remote_database = reinterpretAsIdentifier(args[1], local_context).name; String remote_table = typeid_cast(*args[2]).name; From 854a2f4cc8d733ec2311cc4c403b3e8d60a9ea5e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jun 2015 00:28:04 +0300 Subject: [PATCH 078/109] dbms: added optional section SETTINGS to SELECT query [#METR-16354]. --- .../DB/Interpreters/InterpreterSelectQuery.h | 12 +++++++ .../DB/Interpreters/InterpreterSetQuery.h | 35 +++++++++++++------ dbms/include/DB/Parsers/ASTSelectQuery.h | 1 + dbms/include/DB/Parsers/ParserSetQuery.h | 6 ++++ .../Interpreters/InterpreterSelectQuery.cpp | 30 +++++++++++++--- dbms/src/Parsers/ASTSelectQuery.cpp | 1 + dbms/src/Parsers/ParserSelectQuery.cpp | 17 +++++++++ dbms/src/Parsers/ParserSetQuery.cpp | 20 +++++++---- dbms/src/Parsers/formatAST.cpp | 14 ++++++++ 9 files changed, 114 insertions(+), 22 deletions(-) diff --git a/dbms/include/DB/Interpreters/InterpreterSelectQuery.h b/dbms/include/DB/Interpreters/InterpreterSelectQuery.h index 6c6d278dc6b..a94df417bef 100644 --- a/dbms/include/DB/Interpreters/InterpreterSelectQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterSelectQuery.h @@ -130,6 +130,18 @@ private: void ignoreWithTotals(); + /** Если в запросе SELECT есть секция SETTINGS, то применить настройки из неё и удалить секцию SETTINGS. + * Затем достать настройки из context и поместить их в settings. + * + * Секция SETTINGS - настройки для конкретного запроса. + * Обычно настройки могут быть переданы другими способами, не внутри запроса. + * Но использование такой секции оправдано, если нужно задать настройки для одного подзапроса. + * + * При распределённой обработке запроса, секция SETTINGS не будет передана внутри запроса, + * а настройки будут переданы отдельно, при отправке запроса. + */ + void initSettings(); + ASTPtr query_ptr; ASTSelectQuery & query; Context context; diff --git a/dbms/include/DB/Interpreters/InterpreterSetQuery.h b/dbms/include/DB/Interpreters/InterpreterSetQuery.h index 91ccaf618da..ac8474862a0 100644 --- a/dbms/include/DB/Interpreters/InterpreterSetQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterSetQuery.h @@ -8,7 +8,7 @@ namespace DB { -/** Установить один или несколько параметров, для сессии или глобально. +/** Установить один или несколько параметров, для сессии или глобально... или для текущего запроса. */ class InterpreterSetQuery { @@ -16,17 +16,36 @@ public: InterpreterSetQuery(ASTPtr query_ptr_, Context & context_) : query_ptr(query_ptr_), context(context_) {} + + /** Обычный запрос SET. Задать настройку на сессию или глобальную (если указано GLOBAL). + */ void execute() { ASTSetQuery & ast = typeid_cast(*query_ptr); - Context & target = ast.global ? context.getGlobalContext() : context.getSessionContext(); + executeImpl(ast, target); + } + /** Задать настроку для текущего контекста (контекста запроса). + * Используется для интерпретации секции SETTINGS в запросе SELECT. + */ + void executeForCurrentContext() + { + ASTSetQuery & ast = typeid_cast(*query_ptr); + executeImpl(ast, context); + } + +private: + ASTPtr query_ptr; + Context & context; + + void executeImpl(ASTSetQuery & ast, Context & target) + { /** Значение readonly понимается следующим образом: - * 0 - можно всё. - * 1 - можно делать только запросы на чтение; в том числе, нельзя менять настройки. - * 2 - можно делать только запросы на чтение и можно менять настройки, кроме настройки readonly. - */ + * 0 - можно всё. + * 1 - можно делать только запросы на чтение; в том числе, нельзя менять настройки. + * 2 - можно делать только запросы на чтение и можно менять настройки, кроме настройки readonly. + */ if (context.getSettingsRef().limits.readonly == 1) throw Exception("Cannot execute SET query in readonly mode", ErrorCodes::READONLY); @@ -39,10 +58,6 @@ public: for (ASTSetQuery::Changes::const_iterator it = ast.changes.begin(); it != ast.changes.end(); ++it) target.setSetting(it->name, it->value); } - -private: - ASTPtr query_ptr; - Context & context; }; diff --git a/dbms/include/DB/Parsers/ASTSelectQuery.h b/dbms/include/DB/Parsers/ASTSelectQuery.h index c40fcd52e22..2d5f9cba3a2 100644 --- a/dbms/include/DB/Parsers/ASTSelectQuery.h +++ b/dbms/include/DB/Parsers/ASTSelectQuery.h @@ -51,6 +51,7 @@ public: ASTPtr order_expression_list; ASTPtr limit_offset; ASTPtr limit_length; + ASTPtr settings; ASTPtr next_union_all; /// Следующий запрос SELECT в цепочке UNION ALL, если такой есть }; diff --git a/dbms/include/DB/Parsers/ParserSetQuery.h b/dbms/include/DB/Parsers/ParserSetQuery.h index 5a0dd4d7074..aaa3a4d7f15 100644 --- a/dbms/include/DB/Parsers/ParserSetQuery.h +++ b/dbms/include/DB/Parsers/ParserSetQuery.h @@ -12,9 +12,15 @@ namespace DB */ class ParserSetQuery : public IParserBase { +public: + ParserSetQuery(bool parse_only_internals_ = false) : parse_only_internals(parse_only_internals_) {} + protected: const char * getName() const { return "SET query"; } bool parseImpl(Pos & pos, Pos end, ASTPtr & node, Pos & max_parsed_pos, Expected & expected); + + /// Парсить список name = value пар, без SET [GLOBAL]. + bool parse_only_internals; }; } diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index 63de3f21ba0..f624375fa6f 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -24,6 +24,7 @@ #include #include +#include #include #include #include @@ -39,10 +40,12 @@ InterpreterSelectQuery::~InterpreterSelectQuery() = default; void InterpreterSelectQuery::init(BlockInputStreamPtr input, const Names & required_column_names, const NamesAndTypesList & table_column_names) { - original_max_threads = settings.max_threads; - ProfileEvents::increment(ProfileEvents::SelectQuery); + initSettings(); + + original_max_threads = settings.max_threads; + if (settings.limits.max_subquery_depth && subquery_depth > settings.limits.max_subquery_depth) throw Exception("Too deep subqueries. Maximum: " + toString(settings.limits.max_subquery_depth), ErrorCodes::TOO_DEEP_SUBQUERIES); @@ -174,7 +177,7 @@ void InterpreterSelectQuery::initQueryAnalyzer() InterpreterSelectQuery::InterpreterSelectQuery(ASTPtr query_ptr_, const Context & context_, QueryProcessingStage::Enum to_stage_, size_t subquery_depth_, BlockInputStreamPtr input_, bool is_union_all_head_) : query_ptr(query_ptr_), query(typeid_cast(*query_ptr)), - context(context_), settings(context.getSettings()), to_stage(to_stage_), subquery_depth(subquery_depth_), + context(context_), to_stage(to_stage_), subquery_depth(subquery_depth_), is_first_select_inside_union_all(is_union_all_head_ && !query.next_union_all.isNull()), log(&Logger::get("InterpreterSelectQuery")) { @@ -185,7 +188,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(ASTPtr query_ptr_, const Context const Names & required_column_names_, QueryProcessingStage::Enum to_stage_, size_t subquery_depth_, BlockInputStreamPtr input_) : query_ptr(query_ptr_), query(typeid_cast(*query_ptr)), - context(context_), settings(context.getSettings()), to_stage(to_stage_), subquery_depth(subquery_depth_), + context(context_), to_stage(to_stage_), subquery_depth(subquery_depth_), is_first_select_inside_union_all(!query.next_union_all.isNull()), log(&Logger::get("InterpreterSelectQuery")) { @@ -196,7 +199,7 @@ InterpreterSelectQuery::InterpreterSelectQuery(ASTPtr query_ptr_, const Context const Names & required_column_names_, const NamesAndTypesList & table_column_names, QueryProcessingStage::Enum to_stage_, size_t subquery_depth_, BlockInputStreamPtr input_) : query_ptr(query_ptr_), query(typeid_cast(*query_ptr)), - context(context_), settings(context.getSettings()), to_stage(to_stage_), subquery_depth(subquery_depth_), + context(context_), to_stage(to_stage_), subquery_depth(subquery_depth_), is_first_select_inside_union_all(!query.next_union_all.isNull()), log(&Logger::get("InterpreterSelectQuery")) { @@ -1028,4 +1031,21 @@ void InterpreterSelectQuery::ignoreWithTotals() } +void InterpreterSelectQuery::initSettings() +{ + if (query.settings) + { + InterpreterSetQuery(query.settings, context).executeForCurrentContext(); + + auto it = std::find(query.children.begin(), query.children.end(), query.settings); + if (query.children.end() == it) + throw Exception("Logical error: cannot find query.settings element in query.children", ErrorCodes::LOGICAL_ERROR); + + query.children.erase(it); + query.settings = nullptr; + } + + settings = context.getSettings(); +} + } diff --git a/dbms/src/Parsers/ASTSelectQuery.cpp b/dbms/src/Parsers/ASTSelectQuery.cpp index 3bbeb504a4e..f98ec0fa39b 100644 --- a/dbms/src/Parsers/ASTSelectQuery.cpp +++ b/dbms/src/Parsers/ASTSelectQuery.cpp @@ -164,6 +164,7 @@ ASTPtr ASTSelectQuery::clone() const CLONE(order_expression_list) CLONE(limit_offset) CLONE(limit_length) + CLONE(settings) CLONE(format) CLONE(next_union_all) diff --git a/dbms/src/Parsers/ParserSelectQuery.cpp b/dbms/src/Parsers/ParserSelectQuery.cpp index 0780028792f..91e806ba436 100644 --- a/dbms/src/Parsers/ParserSelectQuery.cpp +++ b/dbms/src/Parsers/ParserSelectQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include @@ -37,6 +38,7 @@ bool ParserSelectQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Pos & max_p ParserString s_having("HAVING", true, true); ParserString s_order("ORDER", true, true); ParserString s_limit("LIMIT", true, true); + ParserString s_settings("SETTINGS", true, true); ParserString s_format("FORMAT", true, true); ParserString s_union("UNION", true, true); ParserString s_all("ALL", true, true); @@ -281,6 +283,19 @@ bool ParserSelectQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Pos & max_p } } + /// SETTINGS key1 = value1, key2 = value2, ... + if (s_settings.ignore(pos, end, max_parsed_pos, expected)) + { + ws.ignore(pos, end); + + ParserSetQuery parser_settings(true); + + if (!parser_settings.parse(pos, end, select_query->settings, max_parsed_pos, expected)) + return false; + + ws.ignore(pos, end); + } + /// FORMAT format_name if (s_format.ignore(pos, end, max_parsed_pos, expected)) { @@ -339,6 +354,8 @@ bool ParserSelectQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Pos & max_p select_query->children.push_back(select_query->limit_offset); if (select_query->limit_length) select_query->children.push_back(select_query->limit_length); + if (select_query->settings) + select_query->children.push_back(select_query->settings); if (select_query->format) select_query->children.push_back(select_query->format); if (select_query->next_union_all) diff --git a/dbms/src/Parsers/ParserSetQuery.cpp b/dbms/src/Parsers/ParserSetQuery.cpp index 79cff3e8ac7..43291df42b0 100644 --- a/dbms/src/Parsers/ParserSetQuery.cpp +++ b/dbms/src/Parsers/ParserSetQuery.cpp @@ -50,18 +50,24 @@ bool ParserSetQuery::parseImpl(Pos & pos, Pos end, ASTPtr & node, Pos & max_pars Pos begin = pos; ParserWhiteSpaceOrComments ws; - ParserString s_set("SET", true, true); - ParserString s_global("GLOBAL", true, true); ParserString s_comma(","); - ws.ignore(pos, end); + bool global = false; - if (!s_set.ignore(pos, end, max_parsed_pos, expected)) - return false; + if (!parse_only_internals) + { + ParserString s_set("SET", true, true); + ParserString s_global("GLOBAL", true, true); - ws.ignore(pos, end); + ws.ignore(pos, end); - bool global = s_global.ignore(pos, end, max_parsed_pos, expected); + if (!s_set.ignore(pos, end, max_parsed_pos, expected)) + return false; + + ws.ignore(pos, end); + + global = s_global.ignore(pos, end, max_parsed_pos, expected); + } ASTSetQuery::Changes changes; diff --git a/dbms/src/Parsers/formatAST.cpp b/dbms/src/Parsers/formatAST.cpp index a720c44c971..8c0e2bfa574 100644 --- a/dbms/src/Parsers/formatAST.cpp +++ b/dbms/src/Parsers/formatAST.cpp @@ -211,6 +211,20 @@ void formatAST(const ASTSelectQuery & ast, std::ostream & s, size_t indent, bo formatAST(*ast.limit_length, s, indent, hilite, one_line); } + if (ast.settings) + { + s << (hilite ? hilite_keyword : "") << nl_or_ws << indent_str << "SETTINGS " << (hilite ? hilite_none : ""); + + const ASTSetQuery & ast_set = typeid_cast(*ast.settings); + for (ASTSetQuery::Changes::const_iterator it = ast_set.changes.begin(); it != ast_set.changes.end(); ++it) + { + if (it != ast_set.changes.begin()) + s << ", "; + + s << it->name << " = " << apply_visitor(FieldVisitorToString(), it->value); + } + } + if (ast.format) { s << (hilite ? hilite_keyword : "") << nl_or_ws << indent_str << "FORMAT " << (hilite ? hilite_none : ""); From 890ee9c0fabc5f12e5972453bfd1b79b717949d5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jun 2015 02:19:15 +0300 Subject: [PATCH 079/109] dbms: removed aggregate function 'debug' [#METR-2944]. --- .../AggregateFunctionDebug.h | 90 ------------------- .../AggregateFunctionFactory.cpp | 6 +- 2 files changed, 1 insertion(+), 95 deletions(-) delete mode 100644 dbms/include/DB/AggregateFunctions/AggregateFunctionDebug.h diff --git a/dbms/include/DB/AggregateFunctions/AggregateFunctionDebug.h b/dbms/include/DB/AggregateFunctions/AggregateFunctionDebug.h deleted file mode 100644 index 11c8e0531ba..00000000000 --- a/dbms/include/DB/AggregateFunctions/AggregateFunctionDebug.h +++ /dev/null @@ -1,90 +0,0 @@ -#include - - -namespace DB -{ - - -/** Сделано в целях отладки. Подлежит удалению. - */ - -struct AggregateFunctionDebugData -{ - UInt32 value; - - AggregateFunctionDebugData() - { - value = 0xAAAAAAAA; - - if (rand() % 1000 == 0) - throw Exception("Test1"); - } - - ~AggregateFunctionDebugData() - { - try - { - if (value == 0xDEADDEAD) - throw Exception("Double free"); - - if (value != 0xAAAAAAAA) - throw Exception("Corruption"); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - std::terminate(); - } - - value = 0xDEADDEAD; - } -}; - -class AggregateFunctionDebug final : public IUnaryAggregateFunction -{ -public: - String getName() const { return "debug"; } - - DataTypePtr getReturnType() const - { - return new DataTypeUInt32; - } - - void setArgument(const DataTypePtr & argument) - { - } - - void addOne(AggregateDataPtr place, const IColumn & column, size_t row_num) const - { - if (rand() % 1000 == 0) - throw Exception("Test2"); - } - - void merge(AggregateDataPtr place, ConstAggregateDataPtr rhs) const - { - if (rand() % 1000 == 0) - throw Exception("Test3"); - } - - void serialize(ConstAggregateDataPtr place, WriteBuffer & buf) const - { - if (rand() % 1000 == 0) - throw Exception("Test4"); - } - - void deserializeMerge(AggregateDataPtr place, ReadBuffer & buf) const - { - if (rand() % 1000 == 0) - throw Exception("Test5"); - } - - void insertResultInto(ConstAggregateDataPtr place, IColumn & to) const - { - if (rand() % 1000 == 0) - throw Exception("Test6"); - - static_cast(to).getData().push_back(123); - } -}; - -} diff --git a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp index 40db1f5f693..39464720135 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/dbms/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include #include @@ -233,9 +232,7 @@ static IAggregateFunction * createAggregateFunctionArgMinMax(const String & name AggregateFunctionPtr AggregateFunctionFactory::get(const String & name, const DataTypes & argument_types, int recursion_level) const { - if (name == "debug") - return new AggregateFunctionDebug; - else if (name == "count") + if (name == "count") return new AggregateFunctionCount; else if (name == "any") return createAggregateFunctionSingleValue(name, argument_types); @@ -697,7 +694,6 @@ const AggregateFunctionFactory::FunctionNames & AggregateFunctionFactory::getFun { static FunctionNames names { - "debug", "count", "any", "anyLast", From 417ab909b8d5a358d3602a0b150bebda3e427c21 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jun 2015 03:28:37 +0300 Subject: [PATCH 080/109] dbms: added functions for manipulation of aggregation states [#METR-16677]. --- .../DB/Functions/FunctionsMiscellaneous.h | 102 ++++++++++++++++++ dbms/src/Functions/FunctionsMiscellaneous.cpp | 3 + 2 files changed, 105 insertions(+) diff --git a/dbms/include/DB/Functions/FunctionsMiscellaneous.h b/dbms/include/DB/Functions/FunctionsMiscellaneous.h index 77b55120b44..de7d94274dd 100644 --- a/dbms/include/DB/Functions/FunctionsMiscellaneous.h +++ b/dbms/include/DB/Functions/FunctionsMiscellaneous.h @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -20,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -58,6 +60,11 @@ namespace DB * bar(x, min, max, width) - рисует полосу из количества символов, пропорционального (x - min) и равного width при x == max. * * version() - возвращает текущую версию сервера в строке. + * + * finalizeAggregation(agg_state) - по состоянию агрегации получить результат. + * + * runningAccumulate(agg_state) - принимает состояния агрегатной функции и возвращает столбец со значениями, + * являющимися результатом накопления этих состояний для множества строк блока, от первой до текущей строки. */ @@ -890,6 +897,7 @@ using FunctionIsFinite = FunctionNumericPredicate; using FunctionIsInfinite = FunctionNumericPredicate; using FunctionIsNaN = FunctionNumericPredicate; + class FunctionVersion : public IFunction { public: @@ -920,4 +928,98 @@ private: } }; + +/** Весьма необычная функция. + * Принимает состояние агрегатной функции (например runningAccumulate(uniqState(UserID))), + * и для каждой строки блока, возвращает результат агрегатной функции по объединению состояний от всех предыдущих строк блока и текущей строки. + * + * То есть, функция зависит от разбиения данных на блоки и от порядка строк в блоке. + */ +class FunctionRunningAccumulate : public IFunction +{ +public: + static constexpr auto name = "runningAccumulate"; + static IFunction * create(const Context & context) { return new FunctionRunningAccumulate; } + + String getName() const override { return name; } + + DataTypePtr getReturnType(const DataTypes & arguments) const override + { + if (arguments.size() != 1) + throw Exception("Function " + getName() + " requires exactly one argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const DataTypeAggregateFunction * type = typeid_cast(&*arguments[0]); + if (!type) + throw Exception("Argument for function " + getName() + " must have type AggregateFunction - state of aggregate function.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return type->getReturnType()->clone(); + } + + void execute(Block & block, const ColumnNumbers & arguments, size_t result) override + { + const ColumnAggregateFunction * column_with_states = typeid_cast(&*block.getByPosition(arguments.at(0)).column); + if (!column_with_states) + throw Exception( + "Illegal column " + block.getByPosition(arguments.at(0)).column->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + AggregateFunctionPtr aggregate_function_ptr = column_with_states->getAggregateFunction(); + const IAggregateFunction & agg_func = *aggregate_function_ptr; + + auto deleter = [&agg_func] (char * ptr) { agg_func.destroy(ptr); free(ptr); }; + std::unique_ptr place { reinterpret_cast(malloc(agg_func.sizeOfData())), deleter }; + + agg_func.create(place.get()); /// Немного не exception-safe. Если здесь выкинется исключение, то зря вызовется destroy. + + ColumnPtr result_column_ptr = agg_func.getReturnType()->createColumn(); + block.getByPosition(result).column = result_column_ptr; + IColumn & result_column = *result_column_ptr; + result_column.reserve(column_with_states->size()); + + const auto & states = column_with_states->getData(); + for (const auto & state_to_add : states) + { + agg_func.merge(place.get(), state_to_add); + agg_func.insertResultInto(place.get(), result_column); + } + } +}; + + +/** Принимает состояние агрегатной функции. Возвращает результат агрегации. + */ +class FunctionFinalizeAggregation : public IFunction +{ +public: + static constexpr auto name = "finalizeAggregation"; + static IFunction * create(const Context & context) { return new FunctionFinalizeAggregation; } + + String getName() const override { return name; } + + DataTypePtr getReturnType(const DataTypes & arguments) const override + { + if (arguments.size() != 1) + throw Exception("Function " + getName() + " requires exactly one argument.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const DataTypeAggregateFunction * type = typeid_cast(&*arguments[0]); + if (!type) + throw Exception("Argument for function " + getName() + " must have type AggregateFunction - state of aggregate function.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return type->getReturnType()->clone(); + } + + void execute(Block & block, const ColumnNumbers & arguments, size_t result) override + { + ColumnAggregateFunction * column_with_states = typeid_cast(&*block.getByPosition(arguments.at(0)).column); + if (!column_with_states) + throw Exception( + "Illegal column " + block.getByPosition(arguments.at(0)).column->getName() + " of first argument of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + block.getByPosition(result).column = column_with_states->convertToValues(); + } +}; + } diff --git a/dbms/src/Functions/FunctionsMiscellaneous.cpp b/dbms/src/Functions/FunctionsMiscellaneous.cpp index b0cbe7b283a..59dd89481f1 100644 --- a/dbms/src/Functions/FunctionsMiscellaneous.cpp +++ b/dbms/src/Functions/FunctionsMiscellaneous.cpp @@ -340,6 +340,9 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); + + factory.registerFunction(); + factory.registerFunction(); } } From 70d40aedc38095865c923392b64386822a135394 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jun 2015 03:43:54 +0300 Subject: [PATCH 081/109] dbms: added tests for functions of aggregation states [#METR-16677]. --- ..._functions_of_aggregation_states.reference | 20 +++++++++++++++++++ .../00166_functions_of_aggregation_states.sql | 1 + 2 files changed, 21 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00166_functions_of_aggregation_states.reference create mode 100644 dbms/tests/queries/0_stateless/00166_functions_of_aggregation_states.sql diff --git a/dbms/tests/queries/0_stateless/00166_functions_of_aggregation_states.reference b/dbms/tests/queries/0_stateless/00166_functions_of_aggregation_states.reference new file mode 100644 index 00000000000..d5b108ae223 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00166_functions_of_aggregation_states.reference @@ -0,0 +1,20 @@ +0 1249975000 1249975000 +1 3749975000 4999950000 +2 6249975000 11249925000 +3 8749975000 19999900000 +4 11249975000 31249875000 +5 13749975000 44999850000 +6 16249975000 61249825000 +7 18749975000 79999800000 +8 21249975000 101249775000 +9 23749975000 124999750000 +10 26249975000 151249725000 +11 28749975000 179999700000 +12 31249975000 211249675000 +13 33749975000 244999650000 +14 36249975000 281249625000 +15 38749975000 319999600000 +16 41249975000 361249575000 +17 43749975000 404999550000 +18 46249975000 451249525000 +19 48749975000 499999500000 diff --git a/dbms/tests/queries/0_stateless/00166_functions_of_aggregation_states.sql b/dbms/tests/queries/0_stateless/00166_functions_of_aggregation_states.sql new file mode 100644 index 00000000000..b73a04e19b9 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00166_functions_of_aggregation_states.sql @@ -0,0 +1 @@ +SELECT k, finalizeAggregation(sum_state), runningAccumulate(sum_state) FROM (SELECT intDiv(number, 50000) AS k, sumState(number) AS sum_state FROM (SELECT number FROM system.numbers LIMIT 1000000) GROUP BY k ORDER BY k); From af3ccde339c01ac0c86acd4749329ddac63de3dc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jun 2015 04:14:56 +0300 Subject: [PATCH 082/109] dbms: section SETTINGS: added tests [#METR-16354]. --- .../queries/0_stateless/00167_settings_inside_query.reference | 4 ++++ .../tests/queries/0_stateless/00167_settings_inside_query.sql | 3 +++ 2 files changed, 7 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00167_settings_inside_query.reference create mode 100644 dbms/tests/queries/0_stateless/00167_settings_inside_query.sql diff --git a/dbms/tests/queries/0_stateless/00167_settings_inside_query.reference b/dbms/tests/queries/0_stateless/00167_settings_inside_query.reference new file mode 100644 index 00000000000..cd62bbbf596 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00167_settings_inside_query.reference @@ -0,0 +1,4 @@ +123 +123 +61 +62 diff --git a/dbms/tests/queries/0_stateless/00167_settings_inside_query.sql b/dbms/tests/queries/0_stateless/00167_settings_inside_query.sql new file mode 100644 index 00000000000..c24375ded00 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00167_settings_inside_query.sql @@ -0,0 +1,3 @@ +SELECT min(number) FROM system.numbers WHERE toUInt64(number % 1000) IN (SELECT DISTINCT blockSize() FROM system.numbers SETTINGS max_block_size = 123, max_rows_to_read = 1000, read_overflow_mode = 'break') SETTINGS max_rows_to_read = 1000000, read_overflow_mode = 'break'; +SELECT * FROM (SELECT DISTINCT blockSize() AS x FROM system.numbers SETTINGS max_block_size = 123, max_rows_to_read = 1000, read_overflow_mode = 'break'); +SELECT x FROM (SELECT DISTINCT blockSize() AS x FROM remote('127.0.0.{1,2}', system.numbers) WHERE number IN (SELECT number * 2 FROM system.numbers SETTINGS max_rows_to_read = 10000, read_overflow_mode = 'break') SETTINGS max_block_size = 123, max_rows_to_read = 1000, read_overflow_mode = 'break') ORDER BY x; From 557b60a19d79cd5710364bd6ffb1c432c375107c Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jun 2015 22:59:16 +0300 Subject: [PATCH 083/109] dbms: fixed tiny error [#METR-15933]. --- dbms/src/Parsers/parseQuery.cpp | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/dbms/src/Parsers/parseQuery.cpp b/dbms/src/Parsers/parseQuery.cpp index 6f0926d70fd..7ebba6e014d 100644 --- a/dbms/src/Parsers/parseQuery.cpp +++ b/dbms/src/Parsers/parseQuery.cpp @@ -64,8 +64,15 @@ static std::string getSyntaxErrorMessage( { message << ":\n\n"; message.write(begin, max_parsed_pos - begin); - message << "\033[41;1m" << *max_parsed_pos << "\033[0m"; /// Ярко-красный фон. - message.write(max_parsed_pos + 1, end - max_parsed_pos - 1); + + size_t bytes_to_hilite = 1; + while (max_parsed_pos + bytes_to_hilite < end + && static_cast(max_parsed_pos[bytes_to_hilite]) >= 0x80 /// UTF-8 + && static_cast(max_parsed_pos[bytes_to_hilite]) <= 0xBF) + ++bytes_to_hilite; + + message << "\033[41;1m" << std::string(max_parsed_pos, bytes_to_hilite) << "\033[0m"; /// Ярко-красный фон. + message.write(max_parsed_pos + bytes_to_hilite, end - max_parsed_pos - bytes_to_hilite); message << "\n\n"; if (expected && *expected && *expected != '.') From 51be2693eece7726e35030c4099dcd2edaaccc29 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 6 Jun 2015 23:00:08 +0300 Subject: [PATCH 084/109] dbms: addition to SETTINGS [#METR-16354]. --- dbms/include/DB/Interpreters/InterpreterSelectQuery.h | 5 +---- dbms/src/Interpreters/InterpreterSelectQuery.cpp | 9 --------- 2 files changed, 1 insertion(+), 13 deletions(-) diff --git a/dbms/include/DB/Interpreters/InterpreterSelectQuery.h b/dbms/include/DB/Interpreters/InterpreterSelectQuery.h index a94df417bef..405691fa966 100644 --- a/dbms/include/DB/Interpreters/InterpreterSelectQuery.h +++ b/dbms/include/DB/Interpreters/InterpreterSelectQuery.h @@ -130,15 +130,12 @@ private: void ignoreWithTotals(); - /** Если в запросе SELECT есть секция SETTINGS, то применить настройки из неё и удалить секцию SETTINGS. + /** Если в запросе SELECT есть секция SETTINGS, то применить настройки из неё. * Затем достать настройки из context и поместить их в settings. * * Секция SETTINGS - настройки для конкретного запроса. * Обычно настройки могут быть переданы другими способами, не внутри запроса. * Но использование такой секции оправдано, если нужно задать настройки для одного подзапроса. - * - * При распределённой обработке запроса, секция SETTINGS не будет передана внутри запроса, - * а настройки будут переданы отдельно, при отправке запроса. */ void initSettings(); diff --git a/dbms/src/Interpreters/InterpreterSelectQuery.cpp b/dbms/src/Interpreters/InterpreterSelectQuery.cpp index f624375fa6f..a1dd0cfc74d 100644 --- a/dbms/src/Interpreters/InterpreterSelectQuery.cpp +++ b/dbms/src/Interpreters/InterpreterSelectQuery.cpp @@ -1034,17 +1034,8 @@ void InterpreterSelectQuery::ignoreWithTotals() void InterpreterSelectQuery::initSettings() { if (query.settings) - { InterpreterSetQuery(query.settings, context).executeForCurrentContext(); - auto it = std::find(query.children.begin(), query.children.end(), query.settings); - if (query.children.end() == it) - throw Exception("Logical error: cannot find query.settings element in query.children", ErrorCodes::LOGICAL_ERROR); - - query.children.erase(it); - query.settings = nullptr; - } - settings = context.getSettings(); } From 97782b485530b2db348ff8a1feb01ace9b683ecf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Jun 2015 07:59:45 +0300 Subject: [PATCH 085/109] dbms: fixed error with memory tracking [#METR-16433]. --- .../DB/Common/HashTable/HashTableAllocator.h | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/dbms/include/DB/Common/HashTable/HashTableAllocator.h b/dbms/include/DB/Common/HashTable/HashTableAllocator.h index d4f74bef0e5..e3d9b462c39 100644 --- a/dbms/include/DB/Common/HashTable/HashTableAllocator.h +++ b/dbms/include/DB/Common/HashTable/HashTableAllocator.h @@ -151,20 +151,18 @@ public: void * realloc(void * buf, size_t old_size, size_t new_size) { + /// Было в stack_memory, там и останется. if (new_size <= N) return buf; + /// Уже не помещалось в stack_memory. if (old_size > N) return HashTableAllocator::realloc(buf, old_size, new_size); - buf = ::malloc(new_size); - if (nullptr == buf) - DB::throwFromErrno("HashTableAllocator: Cannot malloc.", DB::ErrorCodes::CANNOT_ALLOCATE_MEMORY); - - memcpy(buf, stack_memory, old_size); - memset(reinterpret_cast(buf) + old_size, 0, new_size - old_size); - - return buf; + /// Было в stack_memory, но теперь не помещается. + void * new_buf = HashTableAllocator::alloc(new_size); + memcpy(new_buf, buf, old_size); + return new_buf; } }; From 5a60c588bfafac1327c5bc8bb4eda0b52c1f03ba Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Mon, 8 Jun 2015 18:22:04 +0300 Subject: [PATCH 086/109] dbms: ExpressionAnalyzer: redundant null check eliminated, refactoring --- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index ab25424be59..9307b811854 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -1082,13 +1082,11 @@ void ExpressionAnalyzer::getArrayJoinedColumns() if (select_query && select_query->array_join_expression_list) { ASTs & array_join_asts = select_query->array_join_expression_list->children; - for (size_t i = 0; i < array_join_asts .size(); ++i) + for (const auto & ast : array_join_asts) { - ASTPtr ast = array_join_asts [i]; - - String nested_table_name = ast->getColumnName(); - String nested_table_alias = ast->getAliasOrColumnName(); - if (nested_table_alias == nested_table_name && !typeid_cast(&*ast)) + const String nested_table_name = ast->getColumnName(); + const String nested_table_alias = ast->getAliasOrColumnName(); + if (nested_table_alias == nested_table_name && !typeid_cast(&*ast)) throw Exception("No alias for non-trivial value in ARRAY JOIN: " + nested_table_name, ErrorCodes::ALIAS_REQUIRED); if (array_join_alias_to_name.count(nested_table_alias) || aliases.count(nested_table_alias)) @@ -1097,13 +1095,9 @@ void ExpressionAnalyzer::getArrayJoinedColumns() } ASTs & query_asts = select_query->children; - for (size_t i = 0; i < query_asts.size(); ++i) - { - ASTPtr ast = query_asts[i]; - if (select_query && ast == select_query->array_join_expression_list) - continue; - getArrayJoinedColumnsImpl(ast); - } + for (const auto & ast : query_asts) + if (ast != select_query->array_join_expression_list) + getArrayJoinedColumnsImpl(ast); /// Если результат ARRAY JOIN не используется, придется все равно по-ARRAY-JOIN-ить какой-нибудь столбец, /// чтобы получить правильное количество строк. From 432687925cb4bc1acbd6c1b6a06f33d62d5f31b1 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Mon, 8 Jun 2015 19:05:44 +0300 Subject: [PATCH 087/109] dbms: prevent moving array join-ed columns to PREWHERE. No diagnostic for ExpressionAnalyzer yet though [#METR-16525] --- .../MergeTree/MergeTreeWhereOptimizer.h | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h b/dbms/include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h index dcdcb28e8b8..a3b313158bc 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -36,6 +36,7 @@ class MergeTreeWhereOptimizer static constexpr auto max_columns_relative_size = 0.25f; static constexpr auto and_function_name = "and"; static constexpr auto equals_function_name = "equals"; + static constexpr auto array_join_function_name = "arrayJoin"; public: MergeTreeWhereOptimizer(const MergeTreeWhereOptimizer&) = delete; @@ -48,6 +49,7 @@ public: table_columns{toUnorderedSet(data.getColumnsList())}, log{log} { calculateColumnSizes(data, column_names); + determineArrayJoinedNames(select); optimize(select); } @@ -116,7 +118,7 @@ private: SCOPE_EXIT(++idx); - if (hasRestrictedFunctions(condition)) + if (cannotBeMoved(condition)) continue; IdentifierNameSet identifiers{}; @@ -186,7 +188,7 @@ private: auto & condition = select.where_expression; /// do not optimize restricted expressions - if (hasRestrictedFunctions(select.where_expression.get())) + if (cannotBeMoved(select.where_expression.get())) return; IdentifierNameSet identifiers{}; @@ -309,28 +311,49 @@ private: return true; } - /// we assume all AS aliases have been expanded previously - static bool hasRestrictedFunctions(const IAST * ptr) + /** ARRAY JOIN'ed columns as well as arrayJoin() result cannot be used in PREWHERE, therefore expressions + * containing said columns should not be moved to PREWHERE at all. + * We assume all AS aliases have been expanded prior to using this class */ + bool cannotBeMoved(const IAST * ptr) const { if (const auto function_ptr = typeid_cast(ptr)) { /// disallow arrayJoin expressions to be moved to PREWHERE for now - if ("arrayJoin" == function_ptr->name) + if (array_join_function_name == function_ptr->name) return true; } + else if (const auto identifier_ptr = typeid_cast(ptr)) + { + /// disallow moving result of ARRAY JOIN to PREWHERE + if (identifier_ptr->kind == ASTIdentifier::Column) + if (array_joined_names.count(identifier_ptr->name) || + array_joined_names.count(DataTypeNested::extractNestedTableName(identifier_ptr->name))) + return true; + } for (const auto & child : ptr->children) - if (hasRestrictedFunctions(child.get())) + if (cannotBeMoved(child.get())) return true; return false; } + void determineArrayJoinedNames(ASTSelectQuery & select) + { + /// much simplified code from ExpressionAnalyzer::getArrayJoinedColumns() + if (!select.array_join_expression_list) + return; + + for (const auto & ast : select.array_join_expression_list->children) + array_joined_names.emplace(ast->getAliasOrColumnName()); + } + string_set_t primary_key_columns{}; string_set_t table_columns{}; Logger * log; std::unordered_map column_sizes{}; std::size_t total_column_size{}; + NameSet array_joined_names; }; From 0b3c071eb986af5913ac23fb973653c3f2217412 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Jun 2015 23:19:28 +0300 Subject: [PATCH 088/109] dbms: removed useless code [#METR-16739]. --- dbms/include/DB/DataStreams/BlockStreamProfileInfo.h | 2 -- dbms/src/DataStreams/BlockStreamProfileInfo.cpp | 3 --- 2 files changed, 5 deletions(-) diff --git a/dbms/include/DB/DataStreams/BlockStreamProfileInfo.h b/dbms/include/DB/DataStreams/BlockStreamProfileInfo.h index df387e555c5..036aedc3a98 100644 --- a/dbms/include/DB/DataStreams/BlockStreamProfileInfo.h +++ b/dbms/include/DB/DataStreams/BlockStreamProfileInfo.h @@ -29,8 +29,6 @@ struct BlockStreamProfileInfo using BlockStreamProfileInfos = std::vector; BlockStreamProfileInfos nested_infos; - String column_names; - /// Собрать BlockStreamProfileInfo для ближайших в дереве источников с именем name. Пример; собрать все info для PartialSorting stream-ов. void collectInfosForStreamsWithName(const char * name, BlockStreamProfileInfos & res) const; diff --git a/dbms/src/DataStreams/BlockStreamProfileInfo.cpp b/dbms/src/DataStreams/BlockStreamProfileInfo.cpp index a2e61cce364..ee8f5a5a409 100644 --- a/dbms/src/DataStreams/BlockStreamProfileInfo.cpp +++ b/dbms/src/DataStreams/BlockStreamProfileInfo.cpp @@ -52,9 +52,6 @@ void BlockStreamProfileInfo::update(Block & block) ++blocks; rows += block.rowsInFirstColumn(); bytes += block.bytes(); - - if (column_names.empty()) - column_names = block.dumpNames(); } From c94bd2fc09c0e9557b9973b0ce5340154afe2f71 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 8 Jun 2015 23:22:02 +0300 Subject: [PATCH 089/109] dbms: removed useless code [#METR-16739]. --- .../DataStreams/AddingConstColumnBlockInputStream.h | 2 +- .../DB/DataStreams/AddingDefaultBlockInputStream.h | 2 +- .../DB/DataStreams/AggregatingBlockInputStream.h | 2 +- .../DataStreams/AggregatingSortedBlockInputStream.h | 2 +- .../DB/DataStreams/AsynchronousBlockInputStream.h | 2 +- .../DataStreams/CollapsingFinalBlockInputStream.h | 2 +- .../DataStreams/CollapsingSortedBlockInputStream.h | 2 +- .../include/DB/DataStreams/ConcatBlockInputStream.h | 2 +- .../DB/DataStreams/CreatingSetsBlockInputStream.h | 2 +- .../DB/DataStreams/DistinctBlockInputStream.h | 2 +- .../DB/DataStreams/ExpressionBlockInputStream.h | 2 +- .../include/DB/DataStreams/FilterBlockInputStream.h | 2 +- dbms/include/DB/DataStreams/IBlockInputStream.h | 1 - dbms/include/DB/DataStreams/LazyBlockInputStream.h | 2 +- dbms/include/DB/DataStreams/LimitBlockInputStream.h | 2 +- .../DB/DataStreams/MaterializingBlockInputStream.h | 2 +- .../DB/DataStreams/MergeSortingBlockInputStream.h | 4 ++-- .../DataStreams/MergingAggregatedBlockInputStream.h | 2 +- .../DB/DataStreams/MergingSortedBlockInputStream.h | 2 +- .../include/DB/DataStreams/NativeBlockInputStream.h | 2 +- .../DB/DataStreams/NullAndDoCopyBlockInputStream.h | 2 +- dbms/include/DB/DataStreams/NullBlockInputStream.h | 2 +- dbms/include/DB/DataStreams/OneBlockInputStream.h | 2 +- .../ParallelAggregatingBlockInputStream.h | 2 +- .../DB/DataStreams/PartialSortingBlockInputStream.h | 2 +- .../include/DB/DataStreams/RemoteBlockInputStream.h | 2 +- .../DB/DataStreams/RemoveColumnsBlockInputStream.h | 2 +- .../DB/DataStreams/SummingSortedBlockInputStream.h | 2 +- .../DB/DataStreams/TotalsHavingBlockInputStream.h | 2 +- dbms/include/DB/DataStreams/UnionBlockInputStream.h | 2 +- .../include/DB/Dictionaries/MySQLBlockInputStream.h | 2 +- .../DB/Dictionaries/OwningBufferBlockInputStream.h | 2 +- .../Storages/MergeTree/MergeTreeBlockInputStream.h | 2 +- dbms/src/DataStreams/IBlockInputStream.cpp | 13 ++----------- dbms/src/DataStreams/IProfilingBlockInputStream.cpp | 4 ++-- dbms/src/Interpreters/Join.cpp | 2 +- dbms/src/Storages/StorageBuffer.cpp | 2 +- dbms/src/Storages/StorageLog.cpp | 2 +- dbms/src/Storages/StorageMemory.cpp | 2 +- dbms/src/Storages/StorageSystemNumbers.cpp | 2 +- dbms/src/Storages/StorageTinyLog.cpp | 2 +- 41 files changed, 43 insertions(+), 53 deletions(-) diff --git a/dbms/include/DB/DataStreams/AddingConstColumnBlockInputStream.h b/dbms/include/DB/DataStreams/AddingConstColumnBlockInputStream.h index f9909cbdfca..b600857d007 100644 --- a/dbms/include/DB/DataStreams/AddingConstColumnBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AddingConstColumnBlockInputStream.h @@ -25,7 +25,7 @@ public: children.push_back(input_); } - String getName() const override { return "AddingConstColumnBlockInputStream"; } + String getName() const override { return "AddingConstColumn"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h index 0fe52bb9f96..001ad5937ee 100644 --- a/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AddingDefaultBlockInputStream.h @@ -35,7 +35,7 @@ public: { } - String getName() const override { return "AddingDefaultBlockInputStream"; } + String getName() const override { return "AddingDefault"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/AggregatingBlockInputStream.h b/dbms/include/DB/DataStreams/AggregatingBlockInputStream.h index 0187c1281de..be21624dabd 100644 --- a/dbms/include/DB/DataStreams/AggregatingBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AggregatingBlockInputStream.h @@ -32,7 +32,7 @@ public: children.push_back(input_); } - String getName() const override { return "AggregatingBlockInputStream"; } + String getName() const override { return "Aggregating"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/AggregatingSortedBlockInputStream.h b/dbms/include/DB/DataStreams/AggregatingSortedBlockInputStream.h index 14fe81a8dcf..c0b078575ba 100644 --- a/dbms/include/DB/DataStreams/AggregatingSortedBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AggregatingSortedBlockInputStream.h @@ -26,7 +26,7 @@ public: { } - String getName() const override { return "AggregatingSortedBlockInputStream"; } + String getName() const override { return "AggregatingSorted"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/AsynchronousBlockInputStream.h b/dbms/include/DB/DataStreams/AsynchronousBlockInputStream.h index 2e0c725ec73..c743753353f 100644 --- a/dbms/include/DB/DataStreams/AsynchronousBlockInputStream.h +++ b/dbms/include/DB/DataStreams/AsynchronousBlockInputStream.h @@ -25,7 +25,7 @@ public: children.push_back(in_); } - String getName() const override { return "AsynchronousBlockInputStream"; } + String getName() const override { return "Asynchronous"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/CollapsingFinalBlockInputStream.h b/dbms/include/DB/DataStreams/CollapsingFinalBlockInputStream.h index 11334cd4d3b..6692a586de4 100644 --- a/dbms/include/DB/DataStreams/CollapsingFinalBlockInputStream.h +++ b/dbms/include/DB/DataStreams/CollapsingFinalBlockInputStream.h @@ -23,7 +23,7 @@ public: ~CollapsingFinalBlockInputStream(); - String getName() const override { return "CollapsingFinalBlockInputStream"; } + String getName() const override { return "CollapsingFinal"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/CollapsingSortedBlockInputStream.h b/dbms/include/DB/DataStreams/CollapsingSortedBlockInputStream.h index 96799afa5a4..fd9eef68152 100644 --- a/dbms/include/DB/DataStreams/CollapsingSortedBlockInputStream.h +++ b/dbms/include/DB/DataStreams/CollapsingSortedBlockInputStream.h @@ -30,7 +30,7 @@ public: { } - String getName() const override { return "CollapsingSortedBlockInputStream"; } + String getName() const override { return "CollapsingSorted"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/ConcatBlockInputStream.h b/dbms/include/DB/DataStreams/ConcatBlockInputStream.h index f231340b39e..9013f7b06c0 100644 --- a/dbms/include/DB/DataStreams/ConcatBlockInputStream.h +++ b/dbms/include/DB/DataStreams/ConcatBlockInputStream.h @@ -22,7 +22,7 @@ public: current_stream = children.begin(); } - String getName() const override { return "ConcatBlockInputStream"; } + String getName() const override { return "Concat"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/CreatingSetsBlockInputStream.h b/dbms/include/DB/DataStreams/CreatingSetsBlockInputStream.h index 9c63ee9c5a7..0f8ff730156 100644 --- a/dbms/include/DB/DataStreams/CreatingSetsBlockInputStream.h +++ b/dbms/include/DB/DataStreams/CreatingSetsBlockInputStream.h @@ -28,7 +28,7 @@ public: children.push_back(input); } - String getName() const override { return "CreatingSetsBlockInputStream"; } + String getName() const override { return "CreatingSets"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/DistinctBlockInputStream.h b/dbms/include/DB/DataStreams/DistinctBlockInputStream.h index 2789b82747b..bf4199e4598 100644 --- a/dbms/include/DB/DataStreams/DistinctBlockInputStream.h +++ b/dbms/include/DB/DataStreams/DistinctBlockInputStream.h @@ -20,7 +20,7 @@ public: /// Пустой columns_ значит все столбцы. DistinctBlockInputStream(BlockInputStreamPtr input_, const Limits & limits, size_t limit_, Names columns_); - String getName() const override { return "DistinctBlockInputStream"; } + String getName() const override { return "Distinct"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/ExpressionBlockInputStream.h b/dbms/include/DB/DataStreams/ExpressionBlockInputStream.h index 66e9d2be998..a5b028f0663 100644 --- a/dbms/include/DB/DataStreams/ExpressionBlockInputStream.h +++ b/dbms/include/DB/DataStreams/ExpressionBlockInputStream.h @@ -26,7 +26,7 @@ public: children.push_back(input_); } - String getName() const override { return "ExpressionBlockInputStream"; } + String getName() const override { return "Expression"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/FilterBlockInputStream.h b/dbms/include/DB/DataStreams/FilterBlockInputStream.h index dd587919efe..0c8ab1ec9df 100644 --- a/dbms/include/DB/DataStreams/FilterBlockInputStream.h +++ b/dbms/include/DB/DataStreams/FilterBlockInputStream.h @@ -22,7 +22,7 @@ public: FilterBlockInputStream(BlockInputStreamPtr input_, ssize_t filter_column_); FilterBlockInputStream(BlockInputStreamPtr input_, const String & filter_column_name_); - String getName() const override { return "FilterBlockInputStream"; } + String getName() const override { return "Filter"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/IBlockInputStream.h b/dbms/include/DB/DataStreams/IBlockInputStream.h index 4dcfc940f2c..d5e13a13826 100644 --- a/dbms/include/DB/DataStreams/IBlockInputStream.h +++ b/dbms/include/DB/DataStreams/IBlockInputStream.h @@ -51,7 +51,6 @@ public: /** Для вывода дерева преобразований потока данных (плана выполнения запроса). */ virtual String getName() const = 0; - virtual String getShortName() const; /// То же самое, но без BlockInputStream на конце. /** Уникальный идентификатор части конвейера выполнения запроса. * Источники с одинаковым идентификатором считаются идентичными diff --git a/dbms/include/DB/DataStreams/LazyBlockInputStream.h b/dbms/include/DB/DataStreams/LazyBlockInputStream.h index 81793586727..623f009c7f4 100644 --- a/dbms/include/DB/DataStreams/LazyBlockInputStream.h +++ b/dbms/include/DB/DataStreams/LazyBlockInputStream.h @@ -18,7 +18,7 @@ public: LazyBlockInputStream(Generator generator_) : generator(generator_) {} - String getName() const override { return "LazyBlockInputStream"; } + String getName() const override { return "Lazy"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/LimitBlockInputStream.h b/dbms/include/DB/DataStreams/LimitBlockInputStream.h index c820231fd2f..b6cd2e086de 100644 --- a/dbms/include/DB/DataStreams/LimitBlockInputStream.h +++ b/dbms/include/DB/DataStreams/LimitBlockInputStream.h @@ -18,7 +18,7 @@ class LimitBlockInputStream : public IProfilingBlockInputStream public: LimitBlockInputStream(BlockInputStreamPtr input_, size_t limit_, size_t offset_ = 0); - String getName() const override { return "LimitBlockInputStream"; } + String getName() const override { return "Limit"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/MaterializingBlockInputStream.h b/dbms/include/DB/DataStreams/MaterializingBlockInputStream.h index 5a1649567b4..a9a0ba0fd5d 100644 --- a/dbms/include/DB/DataStreams/MaterializingBlockInputStream.h +++ b/dbms/include/DB/DataStreams/MaterializingBlockInputStream.h @@ -18,7 +18,7 @@ public: children.push_back(input_); } - String getName() const override { return "MaterializingBlockInputStream"; } + String getName() const override { return "Materializing"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/MergeSortingBlockInputStream.h b/dbms/include/DB/DataStreams/MergeSortingBlockInputStream.h index 7a51097af10..1e9a38a8e79 100644 --- a/dbms/include/DB/DataStreams/MergeSortingBlockInputStream.h +++ b/dbms/include/DB/DataStreams/MergeSortingBlockInputStream.h @@ -31,7 +31,7 @@ public: MergeSortingBlocksBlockInputStream(Blocks & blocks_, SortDescription & description_, size_t max_merged_block_size_, size_t limit_ = 0); - String getName() const override { return "MergeSortingBlocksBlockInputStream"; } + String getName() const override { return "MergeSortingBlocks"; } String getID() const override { return getName(); } protected: @@ -73,7 +73,7 @@ public: children.push_back(input_); } - String getName() const override { return "MergeSortingBlockInputStream"; } + String getName() const override { return "MergeSorting"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/MergingAggregatedBlockInputStream.h b/dbms/include/DB/DataStreams/MergingAggregatedBlockInputStream.h index 5330585a835..83f66f341bb 100644 --- a/dbms/include/DB/DataStreams/MergingAggregatedBlockInputStream.h +++ b/dbms/include/DB/DataStreams/MergingAggregatedBlockInputStream.h @@ -24,7 +24,7 @@ public: children.push_back(input_); } - String getName() const override { return "MergingAggregatedBlockInputStream"; } + String getName() const override { return "MergingAggregated"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/MergingSortedBlockInputStream.h b/dbms/include/DB/DataStreams/MergingSortedBlockInputStream.h index 66fc321e5d2..35df7cdd891 100644 --- a/dbms/include/DB/DataStreams/MergingSortedBlockInputStream.h +++ b/dbms/include/DB/DataStreams/MergingSortedBlockInputStream.h @@ -26,7 +26,7 @@ public: children.insert(children.end(), inputs_.begin(), inputs_.end()); } - String getName() const override { return "MergingSortedBlockInputStream"; } + String getName() const override { return "MergingSorted"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/NativeBlockInputStream.h b/dbms/include/DB/DataStreams/NativeBlockInputStream.h index cb277a57939..c9810b55385 100644 --- a/dbms/include/DB/DataStreams/NativeBlockInputStream.h +++ b/dbms/include/DB/DataStreams/NativeBlockInputStream.h @@ -18,7 +18,7 @@ public: NativeBlockInputStream(ReadBuffer & istr_, UInt64 server_revision_ = 0) : istr(istr_), server_revision(server_revision_) {} - String getName() const override { return "NativeBlockInputStream"; } + String getName() const override { return "Native"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/NullAndDoCopyBlockInputStream.h b/dbms/include/DB/DataStreams/NullAndDoCopyBlockInputStream.h index 388dfe3e26b..73a4056410c 100644 --- a/dbms/include/DB/DataStreams/NullAndDoCopyBlockInputStream.h +++ b/dbms/include/DB/DataStreams/NullAndDoCopyBlockInputStream.h @@ -22,7 +22,7 @@ public: children.push_back(input_); } - String getName() const override { return "NullAndDoCopyBlockInputStream"; } + String getName() const override { return "NullAndDoCopy"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/NullBlockInputStream.h b/dbms/include/DB/DataStreams/NullBlockInputStream.h index a935971a43c..856867718d6 100644 --- a/dbms/include/DB/DataStreams/NullBlockInputStream.h +++ b/dbms/include/DB/DataStreams/NullBlockInputStream.h @@ -12,7 +12,7 @@ class NullBlockInputStream : public IBlockInputStream { public: Block read() override { return Block(); } - String getName() const override { return "NullBlockInputStream"; } + String getName() const override { return "Null"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/OneBlockInputStream.h b/dbms/include/DB/DataStreams/OneBlockInputStream.h index 18f05415f12..ea4b9adfefb 100644 --- a/dbms/include/DB/DataStreams/OneBlockInputStream.h +++ b/dbms/include/DB/DataStreams/OneBlockInputStream.h @@ -18,7 +18,7 @@ class OneBlockInputStream : public IProfilingBlockInputStream public: OneBlockInputStream(const Block & block_) : block(block_) {} - String getName() const override { return "OneBlockInputStream"; } + String getName() const override { return "One"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/ParallelAggregatingBlockInputStream.h b/dbms/include/DB/DataStreams/ParallelAggregatingBlockInputStream.h index cf44c7d0745..eef66ba1172 100644 --- a/dbms/include/DB/DataStreams/ParallelAggregatingBlockInputStream.h +++ b/dbms/include/DB/DataStreams/ParallelAggregatingBlockInputStream.h @@ -34,7 +34,7 @@ public: children.insert(children.end(), inputs.begin(), inputs.end()); } - String getName() const override { return "ParallelAggregatingBlockInputStream"; } + String getName() const override { return "ParallelAggregating"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/PartialSortingBlockInputStream.h b/dbms/include/DB/DataStreams/PartialSortingBlockInputStream.h index 4247f49a2e6..cf53d6ac3ab 100644 --- a/dbms/include/DB/DataStreams/PartialSortingBlockInputStream.h +++ b/dbms/include/DB/DataStreams/PartialSortingBlockInputStream.h @@ -21,7 +21,7 @@ public: children.push_back(input_); } - String getName() const override { return "PartialSortingBlockInputStream"; } + String getName() const override { return "PartialSorting"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/RemoteBlockInputStream.h b/dbms/include/DB/DataStreams/RemoteBlockInputStream.h index db2c4e77558..90c66edba95 100644 --- a/dbms/include/DB/DataStreams/RemoteBlockInputStream.h +++ b/dbms/include/DB/DataStreams/RemoteBlockInputStream.h @@ -61,7 +61,7 @@ public: } - String getName() const override { return "RemoteBlockInputStream"; } + String getName() const override { return "Remote"; } String getID() const override diff --git a/dbms/include/DB/DataStreams/RemoveColumnsBlockInputStream.h b/dbms/include/DB/DataStreams/RemoveColumnsBlockInputStream.h index 68e971e3a3d..2952eb58642 100644 --- a/dbms/include/DB/DataStreams/RemoveColumnsBlockInputStream.h +++ b/dbms/include/DB/DataStreams/RemoveColumnsBlockInputStream.h @@ -24,7 +24,7 @@ public: children.push_back(input_); } - String getName() const override { return "RemoveColumnsBlockInputStream"; } + String getName() const override { return "RemoveColumns"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h b/dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h index c05bc4dc340..18ad5740fbb 100644 --- a/dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h +++ b/dbms/include/DB/DataStreams/SummingSortedBlockInputStream.h @@ -30,7 +30,7 @@ public: { } - String getName() const override { return "SummingSortedBlockInputStream"; } + String getName() const override { return "SummingSorted"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/TotalsHavingBlockInputStream.h b/dbms/include/DB/DataStreams/TotalsHavingBlockInputStream.h index 8c6d618fcd1..8ccceeda392 100644 --- a/dbms/include/DB/DataStreams/TotalsHavingBlockInputStream.h +++ b/dbms/include/DB/DataStreams/TotalsHavingBlockInputStream.h @@ -28,7 +28,7 @@ public: children.push_back(input_); } - String getName() const override { return "TotalsHavingBlockInputStream"; } + String getName() const override { return "TotalsHaving"; } String getID() const override { diff --git a/dbms/include/DB/DataStreams/UnionBlockInputStream.h b/dbms/include/DB/DataStreams/UnionBlockInputStream.h index b2dc4c4ef76..d3fc8766ae4 100644 --- a/dbms/include/DB/DataStreams/UnionBlockInputStream.h +++ b/dbms/include/DB/DataStreams/UnionBlockInputStream.h @@ -36,7 +36,7 @@ public: children = inputs; } - String getName() const override { return "UnionBlockInputStream"; } + String getName() const override { return "Union"; } String getID() const override { diff --git a/dbms/include/DB/Dictionaries/MySQLBlockInputStream.h b/dbms/include/DB/Dictionaries/MySQLBlockInputStream.h index 0b5e258f7da..b8f4c4bb244 100644 --- a/dbms/include/DB/Dictionaries/MySQLBlockInputStream.h +++ b/dbms/include/DB/Dictionaries/MySQLBlockInputStream.h @@ -90,7 +90,7 @@ public: } } - String getName() const override { return "MySQLBlockInputStream"; } + String getName() const override { return "MySQL"; } String getID() const override { diff --git a/dbms/include/DB/Dictionaries/OwningBufferBlockInputStream.h b/dbms/include/DB/Dictionaries/OwningBufferBlockInputStream.h index cc4c066f9f9..a1b070cb2d8 100644 --- a/dbms/include/DB/Dictionaries/OwningBufferBlockInputStream.h +++ b/dbms/include/DB/Dictionaries/OwningBufferBlockInputStream.h @@ -22,7 +22,7 @@ public: private: Block readImpl() override { return stream->read(); } - String getName() const override { return "OwningBufferBlockInputStream"; } + String getName() const override { return "OwningBuffer"; } String getID() const override { return "OwningBuffer(" + stream->getID() + ")"; } diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h index 2fbc88d4662..da584b0febb 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h @@ -100,7 +100,7 @@ public: setTotalRowsApprox(total_rows); } - String getName() const override { return "MergeTreeBlockInputStream"; } + String getName() const override { return "MergeTree"; } String getID() const override { diff --git a/dbms/src/DataStreams/IBlockInputStream.cpp b/dbms/src/DataStreams/IBlockInputStream.cpp index cdf602920de..0bdbcb69f04 100644 --- a/dbms/src/DataStreams/IBlockInputStream.cpp +++ b/dbms/src/DataStreams/IBlockInputStream.cpp @@ -57,9 +57,9 @@ size_t IBlockInputStream::checkDepthImpl(size_t max_depth, size_t level) const void IBlockInputStream::dumpTree(std::ostream & ostr, size_t indent, size_t multiplier) { /// Не будем отображать в дереве обёртку потока блоков в AsynchronousBlockInputStream. - if (getShortName() != "Asynchronous") + if (getName() != "Asynchronous") { - ostr << String(indent, ' ') << getShortName(); + ostr << String(indent, ' ') << getName(); if (multiplier > 1) ostr << " × " << multiplier; ostr << std::endl; @@ -91,15 +91,6 @@ void IBlockInputStream::dumpTree(std::ostream & ostr, size_t indent, size_t mult } -String IBlockInputStream::getShortName() const -{ - String res = getName(); - if (0 == strcmp(res.c_str() + res.size() - strlen("BlockInputStream"), "BlockInputStream")) - res = res.substr(0, res.size() - strlen("BlockInputStream")); - return res; -} - - BlockInputStreams IBlockInputStream::getLeaves() { BlockInputStreams res; diff --git a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp index c8ae1a3e999..e3a14875e2b 100644 --- a/dbms/src/DataStreams/IProfilingBlockInputStream.cpp +++ b/dbms/src/DataStreams/IProfilingBlockInputStream.cpp @@ -19,7 +19,7 @@ Block IProfilingBlockInputStream::read() if (!info.started) { info.total_stopwatch.start(); - info.stream_name = getShortName(); + info.stream_name = getName(); for (const auto & child : children) if (const IProfilingBlockInputStream * p_child = dynamic_cast(&*child)) @@ -43,7 +43,7 @@ Block IProfilingBlockInputStream::read() Poco::ScopedLock lock(mutex); std::cerr << std::endl; - std::cerr << "[ " << Poco::ThreadNumber::get() << " ]\t" << getShortName() << std::endl; + std::cerr << "[ " << Poco::ThreadNumber::get() << " ]\t" << getName() << std::endl; std::cerr << "[ " << Poco::ThreadNumber::get() << " ]\t"; for (size_t i = 0; i < res.columns(); ++i) diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index d8abb1453dc..38fda859d68 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -723,7 +723,7 @@ public: { } - String getName() const override { return "NonJoinedBlockInputStream"; } + String getName() const override { return "NonJoined"; } String getID() const override { diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index eef6b61b9e8..ba902d27ca4 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -43,7 +43,7 @@ public: BufferBlockInputStream(const Names & column_names_, StorageBuffer::Buffer & buffer_) : column_names(column_names_.begin(), column_names_.end()), buffer(buffer_) {} - String getName() const { return "BufferBlockInputStream"; } + String getName() const { return "Buffer"; } String getID() const { diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index a46419741de..8527fd32f54 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -46,7 +46,7 @@ public: : block_size(block_size_), column_names(column_names_), storage(storage_), mark_number(mark_number_), rows_limit(rows_limit_), current_mark(mark_number_), max_read_buffer_size(max_read_buffer_size_) {} - String getName() const { return "LogBlockInputStream"; } + String getName() const { return "Log"; } String getID() const { diff --git a/dbms/src/Storages/StorageMemory.cpp b/dbms/src/Storages/StorageMemory.cpp index 4da5d343576..00a92eb262d 100644 --- a/dbms/src/Storages/StorageMemory.cpp +++ b/dbms/src/Storages/StorageMemory.cpp @@ -20,7 +20,7 @@ public: MemoryBlockInputStream(const Names & column_names_, BlocksList::iterator begin_, BlocksList::iterator end_) : column_names(column_names_), begin(begin_), end(end_), it(begin) {} - String getName() const { return "MemoryBlockInputStream"; } + String getName() const { return "Memory"; } String getID() const { diff --git a/dbms/src/Storages/StorageSystemNumbers.cpp b/dbms/src/Storages/StorageSystemNumbers.cpp index 3a41abc363f..92879353ed8 100644 --- a/dbms/src/Storages/StorageSystemNumbers.cpp +++ b/dbms/src/Storages/StorageSystemNumbers.cpp @@ -19,7 +19,7 @@ public: NumbersBlockInputStream(size_t block_size_, size_t offset_, size_t step_) : block_size(block_size_), next(offset_), step(step_) {} - String getName() const { return "NumbersBlockInputStream"; } + String getName() const { return "Numbers"; } String getID() const { return "Numbers"; } protected: diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index b854a958883..c8c357ddad5 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -42,7 +42,7 @@ public: TinyLogBlockInputStream(size_t block_size_, const Names & column_names_, StorageTinyLog & storage_, size_t max_read_buffer_size_) : block_size(block_size_), column_names(column_names_), storage(storage_), max_read_buffer_size(max_read_buffer_size_) {} - String getName() const { return "TinyLogBlockInputStream"; } + String getName() const { return "TinyLog"; } String getID() const; From 4fca014e1b4442bba05cdd663379d53538a79555 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Tue, 9 Jun 2015 19:12:51 +0300 Subject: [PATCH 090/109] dbms: reload initially failed dictionaries with exponential backoff [#METR-16702] --- .../include/DB/Dictionaries/CacheDictionary.h | 2 + dbms/include/DB/Dictionaries/FlatDictionary.h | 18 ++- .../DB/Dictionaries/HashedDictionary.h | 17 ++- dbms/include/DB/Dictionaries/IDictionary.h | 2 + .../DB/Interpreters/ExternalDictionaries.h | 14 +- .../src/Interpreters/ExternalDictionaries.cpp | 138 ++++++++++++++---- 6 files changed, 155 insertions(+), 36 deletions(-) diff --git a/dbms/include/DB/Dictionaries/CacheDictionary.h b/dbms/include/DB/Dictionaries/CacheDictionary.h index dcf89bc7d0b..766b4b4fd5a 100644 --- a/dbms/include/DB/Dictionaries/CacheDictionary.h +++ b/dbms/include/DB/Dictionaries/CacheDictionary.h @@ -42,6 +42,8 @@ public: : CacheDictionary{other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime, other.size} {} + std::exception_ptr getCreationException() const override { return {}; } + std::string getName() const override { return name; } std::string getTypeName() const override { return "Cache"; } diff --git a/dbms/include/DB/Dictionaries/FlatDictionary.h b/dbms/include/DB/Dictionaries/FlatDictionary.h index 43629049dd4..629fe2ddd2e 100644 --- a/dbms/include/DB/Dictionaries/FlatDictionary.h +++ b/dbms/include/DB/Dictionaries/FlatDictionary.h @@ -25,8 +25,17 @@ public: source_ptr{std::move(source_ptr)}, dict_lifetime(dict_lifetime) { createAttributes(); - loadData(); - calculateBytesAllocated(); + + try + { + loadData(); + calculateBytesAllocated(); + } + catch (...) + { + creation_exception = std::current_exception(); + } + creation_time = std::chrono::system_clock::now(); } @@ -34,6 +43,8 @@ public: : FlatDictionary{other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime} {} + std::exception_ptr getCreationException() const override { return creation_exception; } + std::string getName() const override { return name; } std::string getTypeName() const override { return "Flat"; } @@ -398,10 +409,11 @@ private: std::size_t bytes_allocated = 0; std::size_t element_count = 0; std::size_t bucket_count = 0; + mutable std::atomic query_count; std::chrono::time_point creation_time; - mutable std::atomic query_count; + std::exception_ptr creation_exception; }; } diff --git a/dbms/include/DB/Dictionaries/HashedDictionary.h b/dbms/include/DB/Dictionaries/HashedDictionary.h index 3eb29c7e8e6..d07a67ae7ed 100644 --- a/dbms/include/DB/Dictionaries/HashedDictionary.h +++ b/dbms/include/DB/Dictionaries/HashedDictionary.h @@ -22,8 +22,17 @@ public: source_ptr{std::move(source_ptr)}, dict_lifetime(dict_lifetime) { createAttributes(); - loadData(); - calculateBytesAllocated(); + + try + { + loadData(); + calculateBytesAllocated(); + } + catch (...) + { + creation_exception = std::current_exception(); + } + creation_time = std::chrono::system_clock::now(); } @@ -31,6 +40,8 @@ public: : HashedDictionary{other.name, other.dict_struct, other.source_ptr->clone(), other.dict_lifetime} {} + std::exception_ptr getCreationException() const override { return creation_exception; } + std::string getName() const override { return name; } std::string getTypeName() const override { return "Hashed"; } @@ -389,6 +400,8 @@ private: mutable std::atomic query_count{}; std::chrono::time_point creation_time; + + std::exception_ptr creation_exception; }; } diff --git a/dbms/include/DB/Dictionaries/IDictionary.h b/dbms/include/DB/Dictionaries/IDictionary.h index 2981c46a816..446e4fe593f 100644 --- a/dbms/include/DB/Dictionaries/IDictionary.h +++ b/dbms/include/DB/Dictionaries/IDictionary.h @@ -24,6 +24,8 @@ class IDictionary public: using id_t = std::uint64_t; + virtual std::exception_ptr getCreationException() const = 0; + virtual std::string getName() const = 0; virtual std::string getTypeName() const = 0; diff --git a/dbms/include/DB/Interpreters/ExternalDictionaries.h b/dbms/include/DB/Interpreters/ExternalDictionaries.h index 4bf27d8bbee..56464a631dc 100644 --- a/dbms/include/DB/Interpreters/ExternalDictionaries.h +++ b/dbms/include/DB/Interpreters/ExternalDictionaries.h @@ -1,23 +1,23 @@ #pragma once +#include #include #include #include #include #include +#include #include #include #include #include #include #include -#include namespace DB { class Context; -class IDictionary; /** Manages user-defined dictionaries. * Monitors configuration file and automatically reloads dictionaries in a separate thread. @@ -50,8 +50,16 @@ private: std::exception_ptr exception; }; + struct failed_dictionary_info final + { + std::unique_ptr dict; + std::chrono::system_clock::time_point next_attempt_time; + std::uint64_t error_count; + }; + std::unordered_map dictionaries; std::unordered_map update_times; + std::unordered_map failed_dictionaries; std::mt19937_64 rnd_engine{getSeed()}; Context & context; @@ -81,7 +89,7 @@ private: { timespec ts; clock_gettime(CLOCK_MONOTONIC, &ts); - return ts.tv_nsec ^ getpid(); + return static_cast(ts.tv_nsec ^ getpid()); } public: diff --git a/dbms/src/Interpreters/ExternalDictionaries.cpp b/dbms/src/Interpreters/ExternalDictionaries.cpp index 8d62fb6aa3f..2f2d8e2cb32 100644 --- a/dbms/src/Interpreters/ExternalDictionaries.cpp +++ b/dbms/src/Interpreters/ExternalDictionaries.cpp @@ -6,6 +6,15 @@ #include #include + +namespace +{ + /// 5 seconds + const auto backoff_initial_sec = 5; + /// 10 minutes + const auto backoff_max_sec = 10 * 60; +} + namespace DB { @@ -41,6 +50,63 @@ void ExternalDictionaries::reloadImpl(const bool throw_on_error) for (const auto & config_path : config_paths) reloadFromFile(config_path, throw_on_error); + /// list of recreated dictionaries to perform delayed removal from unordered_map + std::list recreated_failed_dictionaries; + + /// retry loading failed dictionaries + for (auto & failed_dictionary : failed_dictionaries) + { + if (std::chrono::system_clock::now() < failed_dictionary.second.next_attempt_time) + continue; + + const auto & name = failed_dictionary.first; + + try + { + auto dict_ptr = failed_dictionary.second.dict->clone(); + if (dict_ptr->getCreationException()) + { + /// recalculate next attempt time + std::uniform_int_distribution distribution( + 0, std::exp2(failed_dictionary.second.error_count)); + + failed_dictionary.second.next_attempt_time = std::chrono::system_clock::now() + + std::chrono::seconds{ + std::min(backoff_max_sec, backoff_initial_sec + distribution(rnd_engine)) + }; + + ++failed_dictionary.second.error_count; + } + else + { + const std::lock_guard lock{dictionaries_mutex}; + + const auto dict_it = dictionaries.find(name); + if (dict_it->second.dict) + dict_it->second.dict->set(dict_ptr.release()); + else + dict_it->second.dict = std::make_shared>(dict_ptr.release()); + + /// erase stored exception on success + dict_it->second.exception = std::exception_ptr{}; + + const auto & lifetime = dict_ptr->getLifetime(); + std::uniform_int_distribution distribution{lifetime.min_sec, lifetime.max_sec}; + update_times[name] = std::chrono::system_clock::now() + std::chrono::seconds{distribution(rnd_engine)}; + + recreated_failed_dictionaries.push_back(name); + } + } + catch (...) + { + LOG_ERROR(log, "Failed reloading " << name << " dictionary due to unexpected error"); + } + } + + /// do not undertake further attempts to recreate these dictionaries + for (const auto & name : recreated_failed_dictionaries) + failed_dictionaries.erase(name); + /// periodic update for (auto & dictionary : dictionaries) { @@ -122,10 +188,10 @@ void ExternalDictionaries::reloadFromFile(const std::string & config_path, const } else { - auto it = last_modification_times.find(config_path); - if (it == std::end(last_modification_times)) - it = last_modification_times.emplace(config_path, Poco::Timestamp{0}).first; - auto & config_last_modified = it->second; + auto modification_time_it = last_modification_times.find(config_path); + if (modification_time_it == std::end(last_modification_times)) + modification_time_it = last_modification_times.emplace(config_path, Poco::Timestamp{0}).first; + auto & config_last_modified = modification_time_it->second; const auto last_modified = config_file.getLastModified(); if (last_modified > config_last_modified) @@ -163,12 +229,31 @@ void ExternalDictionaries::reloadFromFile(const std::string & config_path, const continue; } - auto it = dictionaries.find(name); - if (it != std::end(dictionaries)) - if (it->second.origin != config_path) - throw std::runtime_error{"Overriding dictionary from file " + it->second.origin}; + const auto dict_it = dictionaries.find(name); + if (dict_it != std::end(dictionaries)) + if (dict_it->second.origin != config_path) + throw std::runtime_error{"Overriding dictionary from file " + dict_it->second.origin}; auto dict_ptr = DictionaryFactory::instance().create(name, *config, key, context); + if (const auto exception_ptr = dict_ptr->getCreationException()) + { + const auto failed_dict_it = failed_dictionaries.find(name); + if (failed_dict_it != std::end(failed_dictionaries)) + { + failed_dict_it->second = failed_dictionary_info{ + std::move(dict_ptr), + std::chrono::system_clock::now() + std::chrono::seconds{backoff_initial_sec} + }; + } + else + failed_dictionaries.emplace(name, failed_dictionary_info{ + std::move(dict_ptr), + std::chrono::system_clock::now() + std::chrono::seconds{backoff_initial_sec} + }); + + std::rethrow_exception(exception_ptr); + } + if (!dict_ptr->isCached()) { const auto & lifetime = dict_ptr->getLifetime(); @@ -183,42 +268,38 @@ void ExternalDictionaries::reloadFromFile(const std::string & config_path, const } } + const std::lock_guard lock{dictionaries_mutex}; + /// add new dictionary or update an existing version - if (it == std::end(dictionaries)) - { - const std::lock_guard lock{dictionaries_mutex}; + if (dict_it == std::end(dictionaries)) dictionaries.emplace(name, dictionary_info{ std::make_shared>(dict_ptr.release()), config_path }); - } else { - if (it->second.dict) - it->second.dict->set(dict_ptr.release()); + if (dict_it->second.dict) + dict_it->second.dict->set(dict_ptr.release()); else - { - const std::lock_guard lock{dictionaries_mutex}; - it->second.dict = std::make_shared>(dict_ptr.release()); - } + dict_it->second.dict = std::make_shared>(dict_ptr.release()); /// erase stored exception on success - it->second.exception = std::exception_ptr{}; + dict_it->second.exception = std::exception_ptr{}; + failed_dictionaries.erase(name); } } catch (...) { if (!name.empty()) { + const std::lock_guard lock{dictionaries_mutex}; + const auto exception_ptr = std::current_exception(); - const auto it = dictionaries.find(name); - if (it == std::end(dictionaries)) - { - const std::lock_guard lock{dictionaries_mutex}; + const auto dict_it = dictionaries.find(name); + if (dict_it == std::end(dictionaries)) dictionaries.emplace(name, dictionary_info{nullptr, config_path, exception_ptr}); - } else - it->second.exception = exception_ptr; + dict_it->second.exception = exception_ptr; } try @@ -253,16 +334,17 @@ void ExternalDictionaries::reloadFromFile(const std::string & config_path, const MultiVersion::Version ExternalDictionaries::getDictionary(const std::string & name) const { const std::lock_guard lock{dictionaries_mutex}; - const auto it = dictionaries.find(name); + const auto it = dictionaries.find(name); if (it == std::end(dictionaries)) throw Exception{ "No such dictionary: " + name, ErrorCodes::BAD_ARGUMENTS }; - if (!it->second.dict && it->second.exception) - std::rethrow_exception(it->second.exception); + if (!it->second.dict) + it->second.exception ? std::rethrow_exception(it->second.exception) : + throw Exception{"No dictionary", ErrorCodes::LOGICAL_ERROR}; return it->second.dict->get(); } From b64f094d842c588595c0f5e9e208c8ce748d69e8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Jun 2015 21:58:18 +0300 Subject: [PATCH 091/109] dbms: tiny improvement [#METR-2944]. --- dbms/include/DB/DataTypes/DataTypeAggregateFunction.h | 2 +- dbms/include/DB/DataTypes/DataTypeArray.h | 2 +- dbms/include/DB/DataTypes/DataTypeDate.h | 2 +- dbms/include/DB/DataTypes/DataTypeDateTime.h | 2 +- dbms/include/DB/DataTypes/DataTypeExpression.h | 2 +- dbms/include/DB/DataTypes/DataTypeFixedString.h | 2 +- dbms/include/DB/DataTypes/DataTypeNested.h | 2 +- dbms/include/DB/DataTypes/DataTypeSet.h | 2 +- dbms/include/DB/DataTypes/DataTypeString.h | 2 +- dbms/include/DB/DataTypes/DataTypeTuple.h | 2 +- dbms/include/DB/DataTypes/DataTypesNumberFixed.h | 4 ++-- 11 files changed, 12 insertions(+), 12 deletions(-) diff --git a/dbms/include/DB/DataTypes/DataTypeAggregateFunction.h b/dbms/include/DB/DataTypes/DataTypeAggregateFunction.h index 07ee9f432ac..30742f5946f 100644 --- a/dbms/include/DB/DataTypes/DataTypeAggregateFunction.h +++ b/dbms/include/DB/DataTypes/DataTypeAggregateFunction.h @@ -13,7 +13,7 @@ using Poco::SharedPtr; /** Тип - состояние агрегатной функции. * Параметры типа - это агрегатная функция, типы её аргументов и её параметры (для параметрических агрегатных функций). */ -class DataTypeAggregateFunction : public IDataType +class DataTypeAggregateFunction final : public IDataType { private: AggregateFunctionPtr function; diff --git a/dbms/include/DB/DataTypes/DataTypeArray.h b/dbms/include/DB/DataTypes/DataTypeArray.h index de111073468..8dbf99df3da 100644 --- a/dbms/include/DB/DataTypes/DataTypeArray.h +++ b/dbms/include/DB/DataTypes/DataTypeArray.h @@ -9,7 +9,7 @@ namespace DB using Poco::SharedPtr; -class DataTypeArray : public IDataType +class DataTypeArray final : public IDataType { private: /// Тип элементов массивов. diff --git a/dbms/include/DB/DataTypes/DataTypeDate.h b/dbms/include/DB/DataTypes/DataTypeDate.h index fadb19b5a5f..52830bc4ce3 100644 --- a/dbms/include/DB/DataTypes/DataTypeDate.h +++ b/dbms/include/DB/DataTypes/DataTypeDate.h @@ -10,7 +10,7 @@ namespace DB { -class DataTypeDate : public IDataTypeNumberFixed +class DataTypeDate final : public IDataTypeNumberFixed { public: DataTypeDate() {} diff --git a/dbms/include/DB/DataTypes/DataTypeDateTime.h b/dbms/include/DB/DataTypes/DataTypeDateTime.h index 1153173fc45..045ae089a00 100644 --- a/dbms/include/DB/DataTypes/DataTypeDateTime.h +++ b/dbms/include/DB/DataTypes/DataTypeDateTime.h @@ -10,7 +10,7 @@ namespace DB { -class DataTypeDateTime : public IDataTypeNumberFixed +class DataTypeDateTime final : public IDataTypeNumberFixed { public: DataTypeDateTime() {} diff --git a/dbms/include/DB/DataTypes/DataTypeExpression.h b/dbms/include/DB/DataTypes/DataTypeExpression.h index abd23e04523..262f2a99efa 100644 --- a/dbms/include/DB/DataTypes/DataTypeExpression.h +++ b/dbms/include/DB/DataTypes/DataTypeExpression.h @@ -9,7 +9,7 @@ namespace DB /** * Лямбда-выражение. */ -class DataTypeExpression : public IDataTypeDummy +class DataTypeExpression final : public IDataTypeDummy { private: DataTypes argument_types; diff --git a/dbms/include/DB/DataTypes/DataTypeFixedString.h b/dbms/include/DB/DataTypes/DataTypeFixedString.h index faffb0353fa..af2543785ac 100644 --- a/dbms/include/DB/DataTypes/DataTypeFixedString.h +++ b/dbms/include/DB/DataTypes/DataTypeFixedString.h @@ -13,7 +13,7 @@ namespace DB using Poco::SharedPtr; -class DataTypeFixedString : public IDataType +class DataTypeFixedString final : public IDataType { private: size_t n; diff --git a/dbms/include/DB/DataTypes/DataTypeNested.h b/dbms/include/DB/DataTypes/DataTypeNested.h index d6092574762..de5aa81c43f 100644 --- a/dbms/include/DB/DataTypes/DataTypeNested.h +++ b/dbms/include/DB/DataTypes/DataTypeNested.h @@ -9,7 +9,7 @@ namespace DB using Poco::SharedPtr; -class DataTypeNested : public IDataType +class DataTypeNested final : public IDataType { private: /// Имена и типы вложенных массивов. diff --git a/dbms/include/DB/DataTypes/DataTypeSet.h b/dbms/include/DB/DataTypes/DataTypeSet.h index df62e806cd8..1486979648b 100644 --- a/dbms/include/DB/DataTypes/DataTypeSet.h +++ b/dbms/include/DB/DataTypes/DataTypeSet.h @@ -9,7 +9,7 @@ namespace DB /** Тип данных, соответствующий множеству значений в секции IN. * Используется только как промежуточный вариант при вычислении выражений. */ -class DataTypeSet : public IDataTypeDummy +class DataTypeSet final : public IDataTypeDummy { public: std::string getName() const { return "Set"; } diff --git a/dbms/include/DB/DataTypes/DataTypeString.h b/dbms/include/DB/DataTypes/DataTypeString.h index 6804236d7b7..b474a7e6178 100644 --- a/dbms/include/DB/DataTypes/DataTypeString.h +++ b/dbms/include/DB/DataTypes/DataTypeString.h @@ -13,7 +13,7 @@ namespace DB using Poco::SharedPtr; -class DataTypeString : public IDataType +class DataTypeString final : public IDataType { public: using FieldType = String; diff --git a/dbms/include/DB/DataTypes/DataTypeTuple.h b/dbms/include/DB/DataTypes/DataTypeTuple.h index 7932d6853fc..5b0b45db805 100644 --- a/dbms/include/DB/DataTypes/DataTypeTuple.h +++ b/dbms/include/DB/DataTypes/DataTypeTuple.h @@ -15,7 +15,7 @@ namespace DB * Также может быть использовать в качестве столбца - результата выполнения запроса. * Не может быть сохранён в таблицы. */ -class DataTypeTuple : public IDataType +class DataTypeTuple final : public IDataType { private: DataTypes elems; diff --git a/dbms/include/DB/DataTypes/DataTypesNumberFixed.h b/dbms/include/DB/DataTypes/DataTypesNumberFixed.h index e3fbd275579..3ea76b2d920 100644 --- a/dbms/include/DB/DataTypes/DataTypesNumberFixed.h +++ b/dbms/include/DB/DataTypes/DataTypesNumberFixed.h @@ -13,11 +13,11 @@ template struct DataTypeFromFieldType; #define DEFINE_DATA_TYPE_NUMBER_FIXED(TYPE) \ - class DataType ## TYPE : public IDataTypeNumberFixed \ + class DataType ## TYPE final : public IDataTypeNumberFixed \ { \ public: \ std::string getName() const { return #TYPE; } \ - DataTypePtr clone() const { return new DataType ## TYPE; } \ + DataTypePtr clone() const { return new DataType ## TYPE; } \ }; \ \ template <> struct DataTypeFromFieldType \ From 18a8e1f0ca9ffb6395b075cbf3e4aeb431bcab07 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Jun 2015 22:43:06 +0300 Subject: [PATCH 092/109] dbms: StorageBuffer: added missing support for defaults [#METR-16722]. --- dbms/include/DB/Storages/StorageBuffer.h | 12 ++++++++++-- dbms/src/Storages/StorageBuffer.cpp | 18 ++++++++++++++---- dbms/src/Storages/StorageFactory.cpp | 4 +++- 3 files changed, 27 insertions(+), 7 deletions(-) diff --git a/dbms/include/DB/Storages/StorageBuffer.h b/dbms/include/DB/Storages/StorageBuffer.h index 9efb458b858..57098478f89 100644 --- a/dbms/include/DB/Storages/StorageBuffer.h +++ b/dbms/include/DB/Storages/StorageBuffer.h @@ -48,7 +48,11 @@ public: /** num_shards - уровень внутреннего параллелизма (количество независимых буферов) * Буфер сбрасывается, если превышены все минимальные пороги или хотя бы один из максимальных. */ - static StoragePtr create(const std::string & name_, NamesAndTypesListPtr columns_, Context & context_, + static StoragePtr create(const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_); @@ -113,7 +117,11 @@ private: /// Выполняет сброс данных по таймауту. std::thread flush_thread; - StorageBuffer(const std::string & name_, NamesAndTypesListPtr columns_, Context & context_, + StorageBuffer(const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_); diff --git a/dbms/src/Storages/StorageBuffer.cpp b/dbms/src/Storages/StorageBuffer.cpp index ba902d27ca4..2ed9176f2e2 100644 --- a/dbms/src/Storages/StorageBuffer.cpp +++ b/dbms/src/Storages/StorageBuffer.cpp @@ -13,19 +13,29 @@ namespace DB { -StoragePtr StorageBuffer::create(const std::string & name_, NamesAndTypesListPtr columns_, Context & context_, +StoragePtr StorageBuffer::create(const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_) { return (new StorageBuffer{ - name_, columns_, context_, num_shards_, min_thresholds_, max_thresholds_, destination_database_, destination_table_})->thisPtr(); + name_, columns_, materialized_columns_, alias_columns_, column_defaults_, + context_, num_shards_, min_thresholds_, max_thresholds_, destination_database_, destination_table_})->thisPtr(); } -StorageBuffer::StorageBuffer(const std::string & name_, NamesAndTypesListPtr columns_, Context & context_, +StorageBuffer::StorageBuffer(const std::string & name_, NamesAndTypesListPtr columns_, + const NamesAndTypesList & materialized_columns_, + const NamesAndTypesList & alias_columns_, + const ColumnDefaults & column_defaults_, + Context & context_, size_t num_shards_, const Thresholds & min_thresholds_, const Thresholds & max_thresholds_, const String & destination_database_, const String & destination_table_) - : name(name_), columns(columns_), context(context_), + : IStorage{materialized_columns_, alias_columns_, column_defaults_}, + name(name_), columns(columns_), context(context_), num_shards(num_shards_), buffers(num_shards_), min_thresholds(min_thresholds_), max_thresholds(max_thresholds_), destination_database(destination_database_), destination_table(destination_table_), diff --git a/dbms/src/Storages/StorageFactory.cpp b/dbms/src/Storages/StorageFactory.cpp index 9251d2e87b9..da325c58589 100644 --- a/dbms/src/Storages/StorageFactory.cpp +++ b/dbms/src/Storages/StorageFactory.cpp @@ -373,7 +373,9 @@ StoragePtr StorageFactory::get( size_t max_bytes = apply_visitor(FieldVisitorConvertToNumber(), typeid_cast(*args[8]).value); return StorageBuffer::create( - table_name, columns, context, + table_name, columns, + materialized_columns, alias_columns, column_defaults, + context, num_buckets, {min_time, min_rows, min_bytes}, {max_time, max_rows, max_bytes}, destination_database, destination_table); } From c65b2f6a166d3a7cc4b7ce22f637c904485a573f Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Jun 2015 22:51:45 +0300 Subject: [PATCH 093/109] dbms: added test [#METR-16722]. --- .../0_stateless/00168_buffer_defaults.reference | 9 +++++++++ .../queries/0_stateless/00168_buffer_defaults.sql | 12 ++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00168_buffer_defaults.reference create mode 100644 dbms/tests/queries/0_stateless/00168_buffer_defaults.sql diff --git a/dbms/tests/queries/0_stateless/00168_buffer_defaults.reference b/dbms/tests/queries/0_stateless/00168_buffer_defaults.reference new file mode 100644 index 00000000000..3f70f0ee2c4 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00168_buffer_defaults.reference @@ -0,0 +1,9 @@ +EventDate Date +UTCEventTime DateTime +MoscowEventDate Date DEFAULT toDate(UTCEventTime) +EventDate Date +UTCEventTime DateTime +MoscowEventDate Date DEFAULT toDate(UTCEventTime) +2015-06-09 2015-06-09 01:02:03 2015-06-09 +2015-06-09 2015-06-09 01:02:03 2015-06-09 +2015-06-09 2015-06-09 01:02:03 2015-06-09 diff --git a/dbms/tests/queries/0_stateless/00168_buffer_defaults.sql b/dbms/tests/queries/0_stateless/00168_buffer_defaults.sql new file mode 100644 index 00000000000..a42105f6b12 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00168_buffer_defaults.sql @@ -0,0 +1,12 @@ +DROP TABLE IF EXISTS test.mt; +DROP TABLE IF EXISTS test.mt_buffer; +CREATE TABLE test.mt (EventDate Date, UTCEventTime DateTime, MoscowEventDate Date DEFAULT toDate(UTCEventTime)) ENGINE = MergeTree(EventDate, UTCEventTime, 8192); +CREATE TABLE test.mt_buffer AS test.mt ENGINE = Buffer(test, mt, 16, 10, 100, 10000, 1000000, 10000000, 100000000); +DESC TABLE test.mt; +DESC TABLE test.mt_buffer; +INSERT INTO test.mt (EventDate, UTCEventTime) VALUES ('2015-06-09', '2015-06-09 01:02:03'); +SELECT * FROM test.mt_buffer; +INSERT INTO test.mt_buffer (EventDate, UTCEventTime) VALUES ('2015-06-09', '2015-06-09 01:02:03'); +SELECT * FROM test.mt_buffer; +DROP TABLE test.mt_buffer; +DROP TABLE test.mt; From d1df9bd43b11c86e205a0b188d7e1f29d3a578ee Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Jun 2015 23:55:15 +0300 Subject: [PATCH 094/109] dbms: Join: added support for constant keys [#METR-16762]. --- dbms/src/Interpreters/Join.cpp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 38fda859d68..b614c76fa57 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -365,10 +365,21 @@ bool Join::insertFromBlock(const Block & block) size_t keys_size = key_names_right.size(); ConstColumnPlainPtrs key_columns(keys_size); + /// Редкий случай, когда ключи являются константами. Чтобы не поддерживать отдельный код, материализуем их. + Columns materialized_columns; + /// Запоминаем столбцы ключей, с которыми будем работать for (size_t i = 0; i < keys_size; ++i) + { key_columns[i] = block.getByName(key_names_right[i]).column; + if (key_columns[i]->isConst()) + { + materialized_columns.emplace_back(dynamic_cast(*key_columns[i]).convertToFullColumn()); + key_columns[i] = materialized_columns.back(); + } + } + size_t rows = block.rows(); blocks.push_back(block); @@ -378,7 +389,7 @@ bool Join::insertFromBlock(const Block & block) for (const auto & name : key_names_right) stored_block->erase(stored_block->getPositionByName(name)); - /// Редкий случай, когда соединяемые столбцы являеются константами. Чтобы не поддерживать отдельный код, материализуем их. + /// Редкий случай, когда соединяемые столбцы являются константами. Чтобы не поддерживать отдельный код, материализуем их. for (size_t i = 0, size = stored_block->columns(); i < size; ++i) { ColumnPtr col = stored_block->getByPosition(i).column; @@ -515,10 +526,21 @@ void Join::joinBlockImpl(Block & block, const Maps & maps) const size_t keys_size = key_names_left.size(); ConstColumnPlainPtrs key_columns(keys_size); + /// Редкий случай, когда ключи являются константами. Чтобы не поддерживать отдельный код, материализуем их. + Columns materialized_columns; + /// Запоминаем столбцы ключей, с которыми будем работать for (size_t i = 0; i < keys_size; ++i) + { key_columns[i] = block.getByName(key_names_left[i]).column; + if (key_columns[i]->isConst()) + { + materialized_columns.emplace_back(dynamic_cast(*key_columns[i]).convertToFullColumn()); + key_columns[i] = materialized_columns.back(); + } + } + /// Добавляем в блок новые столбцы. size_t num_columns_to_add = sample_block.columns(); ColumnPlainPtrs added_columns(num_columns_to_add); From b417ef015fcaa9e407c292cf02984a6c1abd86db Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 9 Jun 2015 23:56:40 +0300 Subject: [PATCH 095/109] dbms: added test [#METR-16762]. --- .../00169_join_constant_keys.reference | 3 +++ .../0_stateless/00169_join_constant_keys.sql | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00169_join_constant_keys.reference create mode 100644 dbms/tests/queries/0_stateless/00169_join_constant_keys.sql diff --git a/dbms/tests/queries/0_stateless/00169_join_constant_keys.reference b/dbms/tests/queries/0_stateless/00169_join_constant_keys.reference new file mode 100644 index 00000000000..f957808e94b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00169_join_constant_keys.reference @@ -0,0 +1,3 @@ +1 0 999 +2 0 999 +3 0 999 diff --git a/dbms/tests/queries/0_stateless/00169_join_constant_keys.sql b/dbms/tests/queries/0_stateless/00169_join_constant_keys.sql new file mode 100644 index 00000000000..93c98271ca7 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00169_join_constant_keys.sql @@ -0,0 +1,17 @@ +SELECT + key1, + key2, + table_1 +FROM +( + SELECT + arrayJoin([1, 2, 3]) AS key1, + 0 AS key2, + 999 AS table_1 +) ALL INNER JOIN +( + SELECT + arrayJoin([1, 3, 2]) AS key1, + 0 AS key2, + 999 AS table_1 +) USING key2, key1; From 161d30c9023912acd42634cb7ec5599e2979d336 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Jun 2015 00:34:45 +0300 Subject: [PATCH 096/109] dbms: adding checking of types when JOIN (incomplete) [#METR-16762]. --- dbms/include/DB/Interpreters/Join.h | 3 +++ dbms/src/Interpreters/Join.cpp | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/dbms/include/DB/Interpreters/Join.h b/dbms/include/DB/Interpreters/Join.h index 034390e8492..ca9983348d5 100644 --- a/dbms/include/DB/Interpreters/Join.h +++ b/dbms/include/DB/Interpreters/Join.h @@ -250,6 +250,9 @@ private: /// Проверить не превышены ли допустимые размеры множества bool checkSizeLimits() const; + + /// Кинуть исключение, если в блоках не совпадают типы ключей. + void checkTypesOfKeys(const Block & block_left, const Block & block_right) const; }; typedef Poco::SharedPtr JoinPtr; diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index b614c76fa57..3a3c84b19a8 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -650,10 +650,25 @@ void Join::joinBlockImpl(Block & block, const Maps & maps) const } +void Join::checkTypesOfKeys(const Block & block_left, const Block & block_right) const +{ + size_t keys_size = key_names_left.size(); + + for (size_t i = 0; i < keys_size; ++i) + if (block_left.getByName(key_names_left[i]).type->getName() != block_right.getByName(key_names_right[i]).type->getName()) + throw Exception("Type mismatch of columns to JOIN by: " + + key_names_left[i] + " " + block_left.getByName(key_names_left[i]).type->getName() + " at left, " + + key_names_right[i] + " " + block_right.getByName(key_names_right[i]).type->getName() + " at right, ", + ErrorCodes::TYPE_MISMATCH); +} + + void Join::joinBlock(Block & block) const { Poco::ScopedReadRWLock lock(rwlock); +// checkTypesOfKeys(block, sample_block); + if (kind == ASTJoin::Left && strictness == ASTJoin::Any) joinBlockImpl(block, maps_any); else if (kind == ASTJoin::Inner && strictness == ASTJoin::Any) From 9237e308fbbf8b8185111ea5ff77331a5c0cc2d4 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Jun 2015 02:50:22 +0300 Subject: [PATCH 097/109] dbms: JOIN: added checking of types of key columns [#METR-16762]. --- dbms/include/DB/Interpreters/Join.h | 3 ++- dbms/src/Interpreters/Join.cpp | 30 ++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/dbms/include/DB/Interpreters/Join.h b/dbms/include/DB/Interpreters/Join.h index ca9983348d5..9a413e28cf9 100644 --- a/dbms/include/DB/Interpreters/Join.h +++ b/dbms/include/DB/Interpreters/Join.h @@ -222,7 +222,8 @@ private: bool keys_fit_128_bits; Sizes key_sizes; - Block sample_block; + Block sample_block_with_columns_to_add; + Block sample_block_with_keys; Logger * log; diff --git a/dbms/src/Interpreters/Join.cpp b/dbms/src/Interpreters/Join.cpp index 3a3c84b19a8..ce9479e2f91 100644 --- a/dbms/src/Interpreters/Join.cpp +++ b/dbms/src/Interpreters/Join.cpp @@ -339,15 +339,19 @@ void Join::setSampleBlock(const Block & block) /// Выберем, какую структуру данных для множества использовать. init(chooseMethod(key_columns, keys_fit_128_bits, key_sizes)); - sample_block = block; + sample_block_with_columns_to_add = block; - /// Удаляем из sample_block ключевые столбцы, так как они не нужны. + /// Удаляем из sample_block_with_columns_to_add ключевые столбцы. for (const auto & name : key_names_right) - sample_block.erase(sample_block.getPositionByName(name)); - - for (size_t i = 0, size = sample_block.columns(); i < size; ++i) { - auto & column = sample_block.unsafeGetByPosition(i); + size_t pos = sample_block_with_columns_to_add.getPositionByName(name); + sample_block_with_keys.insert(sample_block_with_columns_to_add.unsafeGetByPosition(pos)); + sample_block_with_columns_to_add.erase(pos); + } + + for (size_t i = 0, size = sample_block_with_columns_to_add.columns(); i < size; ++i) + { + auto & column = sample_block_with_columns_to_add.unsafeGetByPosition(i); if (!column.column) column.column = column.type->createColumn(); } @@ -542,14 +546,14 @@ void Join::joinBlockImpl(Block & block, const Maps & maps) const } /// Добавляем в блок новые столбцы. - size_t num_columns_to_add = sample_block.columns(); + size_t num_columns_to_add = sample_block_with_columns_to_add.columns(); ColumnPlainPtrs added_columns(num_columns_to_add); size_t existing_columns = block.columns(); for (size_t i = 0; i < num_columns_to_add; ++i) { - const ColumnWithNameAndType & src_column = sample_block.getByPosition(i); + const ColumnWithNameAndType & src_column = sample_block_with_columns_to_add.getByPosition(i); ColumnWithNameAndType new_column = src_column.cloneEmpty(); block.insert(new_column); added_columns[i] = new_column.column; @@ -658,7 +662,7 @@ void Join::checkTypesOfKeys(const Block & block_left, const Block & block_right) if (block_left.getByName(key_names_left[i]).type->getName() != block_right.getByName(key_names_right[i]).type->getName()) throw Exception("Type mismatch of columns to JOIN by: " + key_names_left[i] + " " + block_left.getByName(key_names_left[i]).type->getName() + " at left, " - + key_names_right[i] + " " + block_right.getByName(key_names_right[i]).type->getName() + " at right, ", + + key_names_right[i] + " " + block_right.getByName(key_names_right[i]).type->getName() + " at right", ErrorCodes::TYPE_MISMATCH); } @@ -667,7 +671,7 @@ void Join::joinBlock(Block & block) const { Poco::ScopedReadRWLock lock(rwlock); -// checkTypesOfKeys(block, sample_block); + checkTypesOfKeys(block, sample_block_with_keys); if (kind == ASTJoin::Left && strictness == ASTJoin::Any) joinBlockImpl(block, maps_any); @@ -703,7 +707,7 @@ void Join::joinTotals(Block & block) const else { /// Будем присоединять пустые totals - из одной строчки со значениями по-умолчанию. - totals_without_keys = sample_block.cloneEmpty(); + totals_without_keys = sample_block_with_columns_to_add.cloneEmpty(); for (size_t i = 0; i < totals_without_keys.columns(); ++i) { @@ -808,12 +812,12 @@ private: } /// Добавляем в блок новые столбцы. - size_t num_columns_right = parent.sample_block.columns(); + size_t num_columns_right = parent.sample_block_with_columns_to_add.columns(); ColumnPlainPtrs columns_right(num_columns_right); for (size_t i = 0; i < num_columns_right; ++i) { - const ColumnWithNameAndType & src_column = parent.sample_block.getByPosition(i); + const ColumnWithNameAndType & src_column = parent.sample_block_with_columns_to_add.getByPosition(i); ColumnWithNameAndType new_column = src_column.cloneEmpty(); block.insert(new_column); columns_right[i] = new_column.column; From b2bfa55a37370dd23a24173a3759443e3273901e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Jun 2015 02:54:37 +0300 Subject: [PATCH 098/109] dbms: fixed tests [#METR-16762]. --- dbms/tests/queries/0_stateless/00118_storage_join.sql | 2 +- dbms/tests/queries/0_stateless/00119_storage_join.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbms/tests/queries/0_stateless/00118_storage_join.sql b/dbms/tests/queries/0_stateless/00118_storage_join.sql index f52e2c7e5c8..7ad702e8ad4 100644 --- a/dbms/tests/queries/0_stateless/00118_storage_join.sql +++ b/dbms/tests/queries/0_stateless/00118_storage_join.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS test.join; -CREATE TABLE test.join (k UInt8, s String) ENGINE = Join(ANY, LEFT, k); +CREATE TABLE test.join (k UInt64, s String) ENGINE = Join(ANY, LEFT, k); USE test; diff --git a/dbms/tests/queries/0_stateless/00119_storage_join.sql b/dbms/tests/queries/0_stateless/00119_storage_join.sql index 934d05f0f65..9a0ead94bd7 100644 --- a/dbms/tests/queries/0_stateless/00119_storage_join.sql +++ b/dbms/tests/queries/0_stateless/00119_storage_join.sql @@ -1,6 +1,6 @@ DROP TABLE IF EXISTS test.join; -CREATE TABLE test.join (s String, x Array(UInt8), k UInt8) ENGINE = Join(ANY, LEFT, k); +CREATE TABLE test.join (s String, x Array(UInt8), k UInt64) ENGINE = Join(ANY, LEFT, k); USE test; From ea4f4420bac82f94689211fdb5e55fdc6680e314 Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Wed, 10 Jun 2015 15:47:27 +0300 Subject: [PATCH 099/109] dbms: replace old lower/upper(UTF8) with vectorized ones [#METR-14764] --- dbms/include/DB/Functions/FunctionsString.h | 123 +++--------------- dbms/src/Functions/FunctionsString.cpp | 4 - .../00170_lower_upper_utf8.reference | 24 ++++ .../0_stateless/00170_lower_upper_utf8.sql | 29 +++++ 4 files changed, 68 insertions(+), 112 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00170_lower_upper_utf8.reference create mode 100644 dbms/tests/queries/0_stateless/00170_lower_upper_utf8.sql diff --git a/dbms/include/DB/Functions/FunctionsString.h b/dbms/include/DB/Functions/FunctionsString.h index 3025aa46942..4ab35f71ef8 100644 --- a/dbms/include/DB/Functions/FunctionsString.h +++ b/dbms/include/DB/Functions/FunctionsString.h @@ -202,46 +202,11 @@ struct LengthUTF8Impl }; -/** Переводит строку в нижний (верхний) регистр, в текущей локали, в однобайтовой кодировке. - */ -template +template struct LowerUpperImpl { static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets, ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets) - { - res_data.resize(data.size()); - res_offsets.assign(offsets); - array(&*data.begin(), &*data.end(), &*res_data.begin()); - } - - static void vector_fixed(const ColumnString::Chars_t & data, size_t n, - ColumnString::Chars_t & res_data) - { - res_data.resize(data.size()); - array(&*data.begin(), &*data.end(), &*res_data.begin()); - } - - static void constant(const std::string & data, std::string & res_data) - { - res_data.resize(data.size()); - array(reinterpret_cast(&*data.begin()), reinterpret_cast(&*data.end()), - reinterpret_cast(&*res_data.begin())); - } - -private: - static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) - { - for (; src < src_end; ++src, ++dst) - *dst = F(*src); - } -}; - -template -struct LowerUpperImplVectorized -{ - static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets, - ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets) { res_data.resize(data.size()); res_offsets.assign(offsets); @@ -348,9 +313,14 @@ inline void UTF8CyrillicToCase(const UInt8 * & src, const UInt8 * const src_end, } }; +/** Если строка содержит текст в кодировке UTF-8 - перевести его в нижний (верхний) регистр. + * Замечание: предполагается, что после перевода символа в другой регистр, + * длина его мультибайтовой последовательности в UTF-8 не меняется. + * Иначе - поведение не определено. + */ template -struct LowerUpperUTF8ImplVectorized +struct LowerUpperUTF8Impl { static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets, ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets) @@ -487,59 +457,6 @@ private: }; -/** Если строка содержит текст в кодировке UTF-8 - перевести его в нижний (верхний) регистр. - * Замечание: предполагается, что после перевода символа в другой регистр, - * длина его мультибайтовой последовательности в UTF-8 не меняется. - * Иначе - поведение не определено. - */ -template -struct LowerUpperUTF8Impl -{ - static void vector(const ColumnString::Chars_t & data, const ColumnString::Offsets_t & offsets, - ColumnString::Chars_t & res_data, ColumnString::Offsets_t & res_offsets) - { - res_data.resize(data.size()); - res_offsets.assign(offsets); - array(&*data.begin(), &*data.end(), &*res_data.begin()); - } - - static void vector_fixed(const ColumnString::Chars_t & data, size_t n, - ColumnString::Chars_t & res_data) - { - res_data.resize(data.size()); - array(&*data.begin(), &*data.end(), &*res_data.begin()); - } - - static void constant(const std::string & data, std::string & res_data) - { - res_data.resize(data.size()); - array(reinterpret_cast(&*data.begin()), reinterpret_cast(&*data.end()), - reinterpret_cast(&*res_data.begin())); - } - -private: - static void array(const UInt8 * src, const UInt8 * src_end, UInt8 * dst) - { - static Poco::UTF8Encoding utf8; - - while (src < src_end) - { - int chars = utf8.convert(F(utf8.convert(src)), dst, src_end - src); - if (chars) - { - src += chars; - dst += chars; - } - else - { - ++src; - ++dst; - } - } - } -}; - - /** Разворачивает строку в байтах. */ struct ReverseImpl @@ -1676,32 +1593,22 @@ struct NameReverseUTF8 { static constexpr auto name = "reverseUTF8"; }; struct NameSubstring { static constexpr auto name = "substring"; }; struct NameSubstringUTF8 { static constexpr auto name = "substringUTF8"; }; -struct NameSSELower { static constexpr auto name = "sse_lower"; }; -struct NameSSEUpper { static constexpr auto name = "sse_upper"; }; -struct NameSSELowerUTF8 { static constexpr auto name = "sse_lowerUTF8"; }; -struct NameSSEUpperUTF8 { static constexpr auto name = "sse_upperUTF8"; }; - typedef FunctionStringOrArrayToT, NameEmpty, UInt8> FunctionEmpty; typedef FunctionStringOrArrayToT, NameNotEmpty, UInt8> FunctionNotEmpty; typedef FunctionStringOrArrayToT FunctionLength; typedef FunctionStringOrArrayToT FunctionLengthUTF8; -typedef FunctionStringToString, NameLower> FunctionLower; -typedef FunctionStringToString, NameUpper> FunctionUpper; -typedef FunctionStringToString, NameLowerUTF8> FunctionLowerUTF8; -typedef FunctionStringToString, NameUpperUTF8> FunctionUpperUTF8; +typedef FunctionStringToString, NameLower> FunctionLower; +typedef FunctionStringToString, NameUpper> FunctionUpper; +typedef FunctionStringToString< + LowerUpperUTF8Impl<'A', 'Z', Poco::Unicode::toLower, UTF8CyrillicToCase>, + NameLowerUTF8> FunctionLowerUTF8; +typedef FunctionStringToString< + LowerUpperUTF8Impl<'a', 'z', Poco::Unicode::toUpper, UTF8CyrillicToCase>, + NameUpperUTF8> FunctionUpperUTF8; typedef FunctionStringToString FunctionReverse; typedef FunctionStringToString FunctionReverseUTF8; typedef FunctionStringNumNumToString FunctionSubstring; typedef FunctionStringNumNumToString FunctionSubstringUTF8; -using FunctionSSELower = FunctionStringToString, NameSSELower>; -using FunctionSSEUpper = FunctionStringToString, NameSSEUpper>; -using FunctionSSELowerUTF8 = FunctionStringToString< - LowerUpperUTF8ImplVectorized<'A', 'Z', Poco::Unicode::toLower, UTF8CyrillicToCase>, - NameSSELowerUTF8>; -using FunctionSSEUpperUTF8 = FunctionStringToString< - LowerUpperUTF8ImplVectorized<'a', 'z', Poco::Unicode::toUpper, UTF8CyrillicToCase>, - NameSSEUpperUTF8>; - } diff --git a/dbms/src/Functions/FunctionsString.cpp b/dbms/src/Functions/FunctionsString.cpp index eadd6bc6884..17cda08cbc3 100644 --- a/dbms/src/Functions/FunctionsString.cpp +++ b/dbms/src/Functions/FunctionsString.cpp @@ -20,10 +20,6 @@ void registerFunctionsString(FunctionFactory & factory) factory.registerFunction(); factory.registerFunction(); factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); - factory.registerFunction(); } } diff --git a/dbms/tests/queries/0_stateless/00170_lower_upper_utf8.reference b/dbms/tests/queries/0_stateless/00170_lower_upper_utf8.reference new file mode 100644 index 00000000000..f202cb75513 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00170_lower_upper_utf8.reference @@ -0,0 +1,24 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/dbms/tests/queries/0_stateless/00170_lower_upper_utf8.sql b/dbms/tests/queries/0_stateless/00170_lower_upper_utf8.sql new file mode 100644 index 00000000000..d3f1c6f6230 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00170_lower_upper_utf8.sql @@ -0,0 +1,29 @@ +select lower('aaaaaaaaaaaaaaa012345789,.!aaaa' as str) = str; +select lowerUTF8('aaaaaaaaaaaaaaa012345789,.!aaaa' as str) = str; +select lower('AaAaAaAaAaAaAaA012345789,.!aAaA') = 'aaaaaaaaaaaaaaa012345789,.!aaaa'; +select lowerUTF8('AaAaAaAaAaAaAaA012345789,.!aAaA') = 'aaaaaaaaaaaaaaa012345789,.!aaaa'; + +select upper('AAAAAAAAAAAAAAA012345789,.!AAAA' as str) = str; +select upperUTF8('AAAAAAAAAAAAAAA012345789,.!AAAA' as str) = str; +select upper('AaAaAaAaAaAaAaA012345789,.!aAaA') = 'AAAAAAAAAAAAAAA012345789,.!AAAA'; +select upperUTF8('AaAaAaAaAaAaAaA012345789,.!aAaA') = 'AAAAAAAAAAAAAAA012345789,.!AAAA'; + +select sum(lower(materialize('aaaaaaaaaaaaaaa012345789,.!aaaa') as str) = str) = count() array join range(16384) as n; +select sum(lowerUTF8(materialize('aaaaaaaaaaaaaaa012345789,.!aaaa') as str) = str) = count() array join range(16384) as n; +select sum(lower(materialize('AaAaAaAaAaAaAaA012345789,.!aAaA')) = materialize('aaaaaaaaaaaaaaa012345789,.!aaaa')) = count() array join range(16384) as n; +select sum(lowerUTF8(materialize('AaAaAaAaAaAaAaA012345789,.!aAaA')) = materialize('aaaaaaaaaaaaaaa012345789,.!aaaa')) = count() array join range(16384) as n; + +select sum(upper(materialize('AAAAAAAAAAAAAAA012345789,.!AAAA') as str) = str) = count() array join range(16384) as n; +select sum(upperUTF8(materialize('AAAAAAAAAAAAAAA012345789,.!AAAA') as str) = str) = count() array join range(16384) as n; +select sum(upper(materialize('AaAaAaAaAaAaAaA012345789,.!aAaA')) = materialize('AAAAAAAAAAAAAAA012345789,.!AAAA')) = count() array join range(16384) as n; +select sum(upperUTF8(materialize('AaAaAaAaAaAaAaA012345789,.!aAaA')) = materialize('AAAAAAAAAAAAAAA012345789,.!AAAA')) = count() array join range(16384) as n; + +select lower('aaaaАБВГAAAAaaAA') = 'aaaaАБВГaaaaaaaa'; +select upper('aaaaАБВГAAAAaaAA') = 'AAAAАБВГAAAAAAAA'; +select lowerUTF8('aaaaАБВГAAAAaaAA') = 'aaaaабвгaaaaaaaa'; +select upperUTF8('aaaaАБВГAAAAaaAA') = 'AAAAАБВГAAAAAAAA'; + +select sum(lower(materialize('aaaaАБВГAAAAaaAA')) = materialize('aaaaАБВГaaaaaaaa')) = count() array join range(16384) as n; +select sum(upper(materialize('aaaaАБВГAAAAaaAA')) = materialize('AAAAАБВГAAAAAAAA')) = count() array join range(16384) as n; +select sum(lowerUTF8(materialize('aaaaАБВГAAAAaaAA')) = materialize('aaaaабвгaaaaaaaa')) = count() array join range(16384) as n; +select sum(upperUTF8(materialize('aaaaАБВГAAAAaaAA')) = materialize('AAAAАБВГAAAAAAAA')) = count() array join range(16384) as n; From c4d1887a6fde50881381ed88b0718143b03f879b Mon Sep 17 00:00:00 2001 From: Andrey Mironov Date: Wed, 10 Jun 2015 17:49:38 +0300 Subject: [PATCH 100/109] dbms: correctly (I hope) determine PREWHERE dependencies for defaults [#METR-16589] --- .../include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h index da584b0febb..2d2b3230fc9 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeBlockInputStream.h @@ -43,9 +43,7 @@ public: { pre_column_names = prewhere_actions->getRequiredColumns(); - /// @todo somehow decide which injected columns belong to PREWHERE, optimizing reads - pre_column_names.insert(std::end(pre_column_names), - std::begin(injected_columns), std::end(injected_columns)); + injectRequiredColumns(pre_column_names); if (pre_column_names.empty()) pre_column_names.push_back(column_names[0]); From bc17c73ee49d18d8b64e79faead97869fa6c22c1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 10 Jun 2015 22:56:57 +0300 Subject: [PATCH 101/109] SipHash: allowed not to call update for empty strings (less than 0.3% performance degradation on all test data) [#METR-16781]. --- dbms/include/DB/Common/SipHash.h | 1 + dbms/src/Common/tests/sip_hash_perf.cpp | 71 +++++++++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 dbms/src/Common/tests/sip_hash_perf.cpp diff --git a/dbms/include/DB/Common/SipHash.h b/dbms/include/DB/Common/SipHash.h index bcfe81b93ed..280d83a9b84 100644 --- a/dbms/include/DB/Common/SipHash.h +++ b/dbms/include/DB/Common/SipHash.h @@ -76,6 +76,7 @@ public: v3 = 0x7465646279746573ULL ^ k1; cnt = 0; + current_word = 0; } void update(const char * data, u64 size) diff --git a/dbms/src/Common/tests/sip_hash_perf.cpp b/dbms/src/Common/tests/sip_hash_perf.cpp new file mode 100644 index 00000000000..dcd397ad5b9 --- /dev/null +++ b/dbms/src/Common/tests/sip_hash_perf.cpp @@ -0,0 +1,71 @@ +#include +#include +#include + +#include +#include +#include +#include + + +/** Тестировать так: + * + * clickhouse-client --query="SELECT SearchPhrase AS k FROM test.hits WHERE k != ''" > phrases.tsv + * clickhouse-client --query="SELECT URL AS k FROM test.hits" > urls.tsv + * clickhouse-client --query="SELECT SearchPhrase AS k FROM test.hits" > phrases_with_empty.tsv + * clickhouse-client --query="SELECT Title AS k FROM test.hits" > titles.tsv + * clickhouse-client --query="SELECT PageCharset AS k FROM test.hits" > charset.tsv + * + * for i in {1..1000}; do ./sip_hash_perf < titles.tsv 2>&1 | grep Processed | grep -oP '\d+\.\d+ rows/sec'; done | awk '{ if ($1 > x) { x = $1; print x } }' + */ + + +int main(int argc, char ** argv) +{ + std::vector data; + DB::ReadBufferFromFileDescriptor in(STDIN_FILENO); + + std::cerr << std::fixed << std::setprecision(3); + + { + Stopwatch watch; + + while (!in.eof()) + { + data.emplace_back(); + DB::readEscapedString(data.back(), in); + DB::assertString("\n", in); + } + + double seconds = watch.elapsedSeconds(); + std::cerr << "Read " + << data.size() << " rows, " + << (in.count() / 1048576.0) << " MiB " + << " in " << seconds << " sec., " + << (data.size() / seconds) << " rows/sec., " + << (in.count() / 1048576.0 / seconds) << " MiB/sec.\n"; + } + + { + size_t res = 0; + Stopwatch watch; + + for (const auto & s : data) + { + SipHash hash; + hash.update(s.data(), s.size()); + res += hash.get64(); + } + + double seconds = watch.elapsedSeconds(); + std::cerr << "Processed " + << data.size() << " rows, " + << (in.count() / 1048576.0) << " MiB " + << " in " << seconds << " sec., " + << (data.size() / seconds) << " rows/sec., " + << (in.count() / 1048576.0 / seconds) << " MiB/sec. " + << "(res = " << res << ")\n"; + } + + return 0; +} From 7f0e6fd3c18ebf9dbb0f7a2d98deec21921b91fb Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 11 Jun 2015 03:35:36 +0300 Subject: [PATCH 102/109] Merge --- .../include/DB/Storages/MergeTree/DiskSpaceMonitor.h | 5 +++-- .../DB/Storages/MergeTree/MergeTreeDataMerger.h | 2 +- dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp | 11 ++++++----- dbms/src/Storages/StorageMergeTree.cpp | 2 +- dbms/src/Storages/StorageReplicatedMergeTree.cpp | 12 +++++++++--- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/dbms/include/DB/Storages/MergeTree/DiskSpaceMonitor.h b/dbms/include/DB/Storages/MergeTree/DiskSpaceMonitor.h index 04a28e996c1..9513090ec7f 100644 --- a/dbms/include/DB/Storages/MergeTree/DiskSpaceMonitor.h +++ b/dbms/include/DB/Storages/MergeTree/DiskSpaceMonitor.h @@ -6,6 +6,7 @@ #include #include #include +#include namespace DB { @@ -116,8 +117,8 @@ public: { size_t free_bytes = getUnreservedFreeSpace(path); if (free_bytes < size) - throw Exception("Not enough free disk space to reserve: " + toString(free_bytes) + " available, " - + toString(size) + " requested", ErrorCodes::NOT_ENOUGH_SPACE); + throw Exception("Not enough free disk space to reserve: " + formatReadableSizeWithBinarySuffix(free_bytes) + " available, " + + formatReadableSizeWithBinarySuffix(size) + " requested", ErrorCodes::NOT_ENOUGH_SPACE); return new Reservation(size); } diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeDataMerger.h b/dbms/include/DB/Storages/MergeTree/MergeTreeDataMerger.h index 59b3638da43..4e0619e3662 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeDataMerger.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeDataMerger.h @@ -49,7 +49,7 @@ public: DiskSpaceMonitor::Reservation * disk_reservation = nullptr); /// Примерное количество места на диске, нужное для мерджа. С запасом. - size_t estimateDiskSpaceForMerge(const MergeTreeData::DataPartsVector & parts); + static size_t estimateDiskSpaceForMerge(const MergeTreeData::DataPartsVector & parts); /** Отменяет все мерджи. Все выполняющиеся сейчас вызовы mergeParts скоро бросят исключение. * Все новые вызовы будут бросать исключения, пока не будет вызван uncancelAll(). diff --git a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp index fb667751464..d75573dd62b 100644 --- a/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp +++ b/dbms/src/Storages/MergeTree/MergeTreeDataMerger.cpp @@ -220,10 +220,12 @@ bool MergeTreeDataMerger::selectPartsToMerge(MergeTreeData::DataPartsVector & pa { disk_space_warning_time = now; LOG_WARNING(log, "Won't merge parts from " << first_part->name << " to " << last_part->name - << " because not enough free space: " << available_disk_space << " free and unreserved " - << "(" << DiskSpaceMonitor::getReservedSpace() << " reserved in " + << " because not enough free space: " + << formatReadableSizeWithBinarySuffix(available_disk_space) << " free and unreserved " + << "(" << formatReadableSizeWithBinarySuffix(DiskSpaceMonitor::getReservedSpace()) << " reserved in " << DiskSpaceMonitor::getReservationCount() << " chunks), " - << cur_sum << " required now (+" << static_cast((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100) + << formatReadableSizeWithBinarySuffix(cur_sum) + << " required now (+" << static_cast((DISK_USAGE_COEFFICIENT_TO_SELECT - 1.0) * 100) << "% on overhead); suppressing similar warnings for the next hour"); } break; @@ -461,9 +463,8 @@ size_t MergeTreeDataMerger::estimateDiskSpaceForMerge(const MergeTreeData::DataP { size_t res = 0; for (const MergeTreeData::DataPartPtr & part : parts) - { res += part->size_in_bytes; - } + return static_cast(res * DISK_USAGE_COEFFICIENT_TO_RESERVE); } diff --git a/dbms/src/Storages/StorageMergeTree.cpp b/dbms/src/Storages/StorageMergeTree.cpp index f9e9f376fe2..2b0a872d7ce 100644 --- a/dbms/src/Storages/StorageMergeTree.cpp +++ b/dbms/src/Storages/StorageMergeTree.cpp @@ -229,7 +229,7 @@ bool StorageMergeTree::merge(size_t aio_threshold, bool aggressive, BackgroundPr return false; } - merging_tagger = new CurrentlyMergingPartsTagger(parts, merger.estimateDiskSpaceForMerge(parts), *this); + merging_tagger = new CurrentlyMergingPartsTagger(parts, MergeTreeDataMerger::estimateDiskSpaceForMerge(parts), *this); /// Если собираемся сливать большие куски, увеличим счетчик потоков, сливающих большие куски. if (pool_context) diff --git a/dbms/src/Storages/StorageReplicatedMergeTree.cpp b/dbms/src/Storages/StorageReplicatedMergeTree.cpp index e113fc298d3..1cd57a70216 100644 --- a/dbms/src/Storages/StorageReplicatedMergeTree.cpp +++ b/dbms/src/Storages/StorageReplicatedMergeTree.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -860,12 +861,15 @@ bool StorageReplicatedMergeTree::executeLogEntry(const LogEntry & entry, Backgro } } + size_t sum_parts_size_in_bytes = MergeTreeDataMerger::estimateDiskSpaceForMerge(parts); + DiskSpaceMonitor::ReservationPtr reserved_space = DiskSpaceMonitor::reserve(full_path, sum_parts_size_in_bytes); /// Может бросить исключение. + auto table_lock = lockStructure(false); const auto & merge_entry = context.getMergeList().insert(database_name, table_name, entry.new_part_name); MergeTreeData::Transaction transaction; size_t aio_threshold = context.getSettings().min_bytes_to_use_direct_io; - MergeTreeData::DataPartPtr part = merger.mergeParts(parts, entry.new_part_name, *merge_entry, aio_threshold, &transaction); + MergeTreeData::DataPartPtr part = merger.mergeParts(parts, entry.new_part_name, *merge_entry, aio_threshold, &transaction, reserved_space); zkutil::Ops ops; checkPartAndAddToZooKeeper(part, ops); @@ -1360,8 +1364,10 @@ void StorageReplicatedMergeTree::mergeSelectingThread() String merged_name; - if ( !merger.selectPartsToMerge(parts, merged_name, MergeTreeDataMerger::NO_LIMIT, false, false, only_small, can_merge) - && !merger.selectPartsToMerge(parts, merged_name, MergeTreeDataMerger::NO_LIMIT, true, false, only_small, can_merge)) + size_t disk_space = DiskSpaceMonitor::getUnreservedFreeSpace(full_path); + + if ( !merger.selectPartsToMerge(parts, merged_name, disk_space, false, false, only_small, can_merge) + && !merger.selectPartsToMerge(parts, merged_name, disk_space, true, false, only_small, can_merge)) { break; } From 84c23517745d207c68617beb0b13f5c5f97a6b15 Mon Sep 17 00:00:00 2001 From: Pavel Kartavyy Date: Tue, 26 May 2015 15:44:34 +0300 Subject: [PATCH 103/109] remove old todo --- dbms/src/Interpreters/InterpreterCheckQuery.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Interpreters/InterpreterCheckQuery.cpp b/dbms/src/Interpreters/InterpreterCheckQuery.cpp index 8bbe323c835..cc85f33c74c 100644 --- a/dbms/src/Interpreters/InterpreterCheckQuery.cpp +++ b/dbms/src/Interpreters/InterpreterCheckQuery.cpp @@ -12,7 +12,6 @@ InterpreterCheckQuery::InterpreterCheckQuery(DB::ASTPtr query_ptr_, DB::Context& BlockInputStreamPtr InterpreterCheckQuery::execute() { - /// @TODO ASTCheckQuery & alter = typeid_cast(*query_ptr); String & table_name = alter.table; String database_name = alter.database.empty() ? context.getCurrentDatabase() : alter.database; From bff45a25fad7dac0adf37f871a3ae16a6fc0396d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2015 02:09:56 +0300 Subject: [PATCH 104/109] dbms: improved type checking in IN [#METR-16793]. --- dbms/src/Interpreters/Set.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/dbms/src/Interpreters/Set.cpp b/dbms/src/Interpreters/Set.cpp index 8c4a9dc10f6..7f0fb917923 100644 --- a/dbms/src/Interpreters/Set.cpp +++ b/dbms/src/Interpreters/Set.cpp @@ -325,8 +325,21 @@ static Field convertToType(const Field & src, const IDataType & type) + Field::Types::toString(src.getType()) + " literal at right"); } } + else + { + if (src.getType() == Field::Types::UInt64 + || src.getType() == Field::Types::Int64 + || src.getType() == Field::Types::Float64 + || src.getType() == Field::Types::Null + || (src.getType() == Field::Types::String + && !typeid_cast(&type) + && !typeid_cast(&type)) + || (src.getType() == Field::Types::Array + && !typeid_cast(&type))) + throw Exception("Type mismatch in IN section: " + type.getName() + " at left, " + + Field::Types::toString(src.getType()) + " literal at right"); + } - /// В остальных случаях, приведение типа не осуществляется. return src; } From 5b8008e2b1506e29d877621abe9405156e83b4c8 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2015 02:38:55 +0300 Subject: [PATCH 105/109] dbms: fixed error with tuples [#METR-13649]. --- dbms/include/DB/DataTypes/DataTypeTuple.h | 1 + .../queries/0_stateless/00171_array_of_tuple_remote.reference | 2 ++ dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.sql | 1 + 3 files changed, 4 insertions(+) create mode 100644 dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.reference create mode 100644 dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.sql diff --git a/dbms/include/DB/DataTypes/DataTypeTuple.h b/dbms/include/DB/DataTypes/DataTypeTuple.h index 5b0b45db805..db9dd5bdb09 100644 --- a/dbms/include/DB/DataTypes/DataTypeTuple.h +++ b/dbms/include/DB/DataTypes/DataTypeTuple.h @@ -138,6 +138,7 @@ public: { ColumnWithNameAndType col; col.column = elems[i]->createColumn(); + col.type = elems[i]->clone(); tuple_block.insert(col); } return new ColumnTuple(tuple_block); diff --git a/dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.reference b/dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.reference new file mode 100644 index 00000000000..cd4823e219f --- /dev/null +++ b/dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.reference @@ -0,0 +1,2 @@ +[(1,4),(2,5),(3,6)] +[(1,4),(2,5),(3,6)] diff --git a/dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.sql b/dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.sql new file mode 100644 index 00000000000..5db737e8e8b --- /dev/null +++ b/dbms/tests/queries/0_stateless/00171_array_of_tuple_remote.sql @@ -0,0 +1 @@ +SELECT arrayMap((x, y) -> (x, y), [1, 2, 3], [4, 5, 6]) FROM remote('127.0.0.{1,2}', system.one) ORDER BY rand(); From a9becd36ea876a2a5ae63ba37d0a73e2dbbfbd21 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2015 08:18:47 +0300 Subject: [PATCH 106/109] dbms: allowed constant expressions in IN [#METR-2944]. --- dbms/include/DB/Interpreters/Set.h | 2 +- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 2 +- dbms/src/Interpreters/Set.cpp | 76 +++++++++++++------- 3 files changed, 54 insertions(+), 26 deletions(-) diff --git a/dbms/include/DB/Interpreters/Set.h b/dbms/include/DB/Interpreters/Set.h index 4b6bfce1d75..e2dc2405279 100644 --- a/dbms/include/DB/Interpreters/Set.h +++ b/dbms/include/DB/Interpreters/Set.h @@ -283,7 +283,7 @@ public: * node - это список значений: 1, 2, 3 или список tuple-ов: (1, 2), (3, 4), (5, 6). * create_ordered_set - создавать ли вектор упорядоченных элементов. Нужен для работы индекса */ - void createFromAST(DataTypes & types, ASTPtr node, bool create_ordered_set); + void createFromAST(DataTypes & types, ASTPtr node, const Context & context, bool create_ordered_set); // Возвращает false, если превышено какое-нибудь ограничение, и больше не нужно вставлять. bool insertFromBlock(const Block & block, bool create_ordered_set = false); diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 9307b811854..8906e36fbfa 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -954,7 +954,7 @@ void ExpressionAnalyzer::makeExplicitSet(ASTFunction * node, const Block & sampl ASTPtr ast_set_ptr = ast_set; ast_set->set = new Set(settings.limits); ast_set->is_explicit = true; - ast_set->set->createFromAST(set_element_types, elements_ast, create_ordered_set); + ast_set->set->createFromAST(set_element_types, elements_ast, context, create_ordered_set); arg = ast_set_ptr; } diff --git a/dbms/src/Interpreters/Set.cpp b/dbms/src/Interpreters/Set.cpp index 7f0fb917923..8e74ee231ef 100644 --- a/dbms/src/Interpreters/Set.cpp +++ b/dbms/src/Interpreters/Set.cpp @@ -10,8 +10,12 @@ #include #include #include +#include #include +#include +#include + #include #include #include @@ -259,10 +263,10 @@ static Field convertToType(const Field & src, const IDataType & type) if (is_uint8 || is_uint16 || is_uint32 || is_uint64) { if (src.getType() == Field::Types::Int64) - throw Exception("Type mismatch in IN section: " + type.getName() + " at left, signed literal at right"); + throw Exception("Type mismatch in IN section: " + type.getName() + " at left, signed at right"); if (src.getType() == Field::Types::Float64) - throw Exception("Type mismatch in IN section: " + type.getName() + " at left, floating point literal at right"); + throw Exception("Type mismatch in IN section: " + type.getName() + " at left, floating point at right"); if (src.getType() == Field::Types::UInt64) { @@ -276,12 +280,12 @@ static Field convertToType(const Field & src, const IDataType & type) } throw Exception("Type mismatch in IN section: " + type.getName() + " at left, " - + Field::Types::toString(src.getType()) + " literal at right"); + + Field::Types::toString(src.getType()) + " at right"); } else if (is_int8 || is_int16 || is_int32 || is_int64) { if (src.getType() == Field::Types::Float64) - throw Exception("Type mismatch in IN section: " + type.getName() + " at left, floating point literal at right"); + throw Exception("Type mismatch in IN section: " + type.getName() + " at left, floating point at right"); if (src.getType() == Field::Types::UInt64) { @@ -308,7 +312,7 @@ static Field convertToType(const Field & src, const IDataType & type) } throw Exception("Type mismatch in IN section: " + type.getName() + " at left, " - + Field::Types::toString(src.getType()) + " literal at right"); + + Field::Types::toString(src.getType()) + " at right"); } else if (is_float32 || is_float64) { @@ -322,7 +326,7 @@ static Field convertToType(const Field & src, const IDataType & type) return src; throw Exception("Type mismatch in IN section: " + type.getName() + " at left, " - + Field::Types::toString(src.getType()) + " literal at right"); + + Field::Types::toString(src.getType()) + " at right"); } } else @@ -337,22 +341,54 @@ static Field convertToType(const Field & src, const IDataType & type) || (src.getType() == Field::Types::Array && !typeid_cast(&type))) throw Exception("Type mismatch in IN section: " + type.getName() + " at left, " - + Field::Types::toString(src.getType()) + " literal at right"); + + Field::Types::toString(src.getType()) + " at right"); } return src; } -void Set::createFromAST(DataTypes & types, ASTPtr node, bool create_ordered_set) +/** Выполнить константное выражение (для элемента множества в IN). Весьма неоптимально. */ +static Field evaluateConstantExpression(ASTPtr & node, const Context & context) { - /** NOTE: - * На данный момент в секции IN не поддерживаются выражения (вызовы функций), кроме кортежей. - * То есть, например, не поддерживаются массивы. А по хорошему, хотелось бы поддерживать. - * Для этого можно сделать constant folding с помощью ExpressionAnalyzer/ExpressionActions. - * Но при этом, конечно же, не забыть про производительность работы с крупными множествами. - */ + ExpressionActionsPtr expr_for_constant_folding = ExpressionAnalyzer( + node, context, NamesAndTypesList{{ "_dummy", new DataTypeUInt8 }}).getConstActions(); + /// В блоке должен быть хотя бы один столбец, чтобы у него было известно число строк. + Block block_with_constants{{ new ColumnConstUInt8(1, 0), new DataTypeUInt8, "_dummy" }}; + + expr_for_constant_folding->execute(block_with_constants); + + if (!block_with_constants || block_with_constants.rows() == 0) + throw Exception("Logical error: empty block after evaluation constant expression for IN", ErrorCodes::LOGICAL_ERROR); + + String name = node->getColumnName(); + + if (!block_with_constants.has(name)) + throw Exception("Element of set in IN is not a constant expression: " + name, ErrorCodes::BAD_ARGUMENTS); + + const IColumn & result_column = *block_with_constants.getByName(name).column; + + if (!result_column.isConst()) + throw Exception("Element of set in IN is not a constant expression: " + name, ErrorCodes::BAD_ARGUMENTS); + + return result_column[0]; +} + + +static Field extractValueFromNode(ASTPtr & node, const IDataType & type, const Context & context) +{ + if (ASTLiteral * lit = typeid_cast(node.get())) + return convertToType(lit->value, type); + else if (typeid_cast(node.get())) + return convertToType(evaluateConstantExpression(node, context), type); + else + throw Exception("Incorrect element of set. Must be literal or constant expression.", ErrorCodes::INCORRECT_ELEMENT_OF_SET); +} + + +void Set::createFromAST(DataTypes & types, ASTPtr node, const Context & context, bool create_ordered_set) +{ data_types = types; /// Засунем множество в блок. @@ -372,10 +408,7 @@ void Set::createFromAST(DataTypes & types, ASTPtr node, bool create_ordered_set) { if (data_types.size() == 1) { - if (ASTLiteral * lit = typeid_cast(&**it)) - block.getByPosition(0).column->insert(convertToType(lit->value, *data_types[0])); - else - throw Exception("Incorrect element of set. Must be literal.", ErrorCodes::INCORRECT_ELEMENT_OF_SET); + block.getByPosition(0).column->insert(extractValueFromNode(*it, *data_types[0], context)); } else if (ASTFunction * func = typeid_cast(&**it)) { @@ -388,16 +421,11 @@ void Set::createFromAST(DataTypes & types, ASTPtr node, bool create_ordered_set) for (size_t j = 0; j < tuple_size; ++j) { - if (ASTLiteral * lit = typeid_cast(&*func->arguments->children[j])) - block.getByPosition(j).column->insert(convertToType(lit->value, *data_types[j])); - else - throw Exception("Incorrect element of tuple in set. Must be literal.", ErrorCodes::INCORRECT_ELEMENT_OF_SET); + block.getByPosition(j).column->insert(extractValueFromNode(func->arguments->children[j], *data_types[j], context)); } } else throw Exception("Incorrect element of set", ErrorCodes::INCORRECT_ELEMENT_OF_SET); - - /// NOTE: Потом можно реализовать возможность задавать константные выражения в множествах. } if (create_ordered_set) From 779d85afc90fe44059fd68e9a7de11289e548901 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2015 08:19:18 +0300 Subject: [PATCH 107/109] dbms: addition to prev. revision [#METR-2944]. --- dbms/src/Interpreters/Set.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/dbms/src/Interpreters/Set.cpp b/dbms/src/Interpreters/Set.cpp index 8e74ee231ef..6bb858a5aaf 100644 --- a/dbms/src/Interpreters/Set.cpp +++ b/dbms/src/Interpreters/Set.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include From 7b76ff3118d5dc5920ea792e3775f6ebaac91cd0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2015 08:54:49 +0300 Subject: [PATCH 108/109] dbms: additions to prev. revision [#METR-2944]. --- dbms/src/Interpreters/ExpressionAnalyzer.cpp | 26 ++++++++++++------- dbms/src/Interpreters/Set.cpp | 19 ++++++++++++-- .../00172_constexprs_in_set.reference | 6 +++++ .../0_stateless/00172_constexprs_in_set.sql | 6 +++++ 4 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00172_constexprs_in_set.reference create mode 100644 dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql diff --git a/dbms/src/Interpreters/ExpressionAnalyzer.cpp b/dbms/src/Interpreters/ExpressionAnalyzer.cpp index 8906e36fbfa..5b4d58b65f8 100644 --- a/dbms/src/Interpreters/ExpressionAnalyzer.cpp +++ b/dbms/src/Interpreters/ExpressionAnalyzer.cpp @@ -922,16 +922,24 @@ void ExpressionAnalyzer::makeExplicitSet(ASTFunction * node, const Block & sampl if (ASTFunction * set_func = typeid_cast(&*arg)) { - if (set_func->name != "tuple") - throw Exception("Incorrect type of 2nd argument for function " + node->name + ". Must be subquery or set of values.", - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - - /// Отличм случай (x, y) in ((1, 2), (3, 4)) от случая (x, y) in (1, 2). - ASTFunction * any_element = typeid_cast(&*set_func->arguments->children.at(0)); - if (set_element_types.size() >= 2 && (!any_element || any_element->name != "tuple")) - single_value = true; + if (set_func->name == "tuple") + { + /// Отличм случай (x, y) in ((1, 2), (3, 4)) от случая (x, y) in (1, 2). + ASTFunction * any_element = typeid_cast(&*set_func->arguments->children.at(0)); + if (set_element_types.size() >= 2 && (!any_element || any_element->name != "tuple")) + single_value = true; + else + elements_ast = set_func->arguments; + } else - elements_ast = set_func->arguments; + { + if (set_element_types.size() >= 2) + throw Exception("Incorrect type of 2nd argument for function " + node->name + + ". Must be subquery or set of " + toString(set_element_types.size()) + "-element tuples.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + single_value = true; + } } else if (typeid_cast(&*arg)) { diff --git a/dbms/src/Interpreters/Set.cpp b/dbms/src/Interpreters/Set.cpp index 6bb858a5aaf..08a6c6e0564 100644 --- a/dbms/src/Interpreters/Set.cpp +++ b/dbms/src/Interpreters/Set.cpp @@ -19,6 +19,8 @@ #include #include #include +#include +#include namespace DB @@ -234,7 +236,7 @@ bool Set::insertFromBlock(const Block & block, bool create_ordered_set) */ static Field convertToType(const Field & src, const IDataType & type) { - if (type.behavesAsNumber()) + if (type.isNumeric()) { bool is_uint8 = false; bool is_uint16 = false; @@ -246,6 +248,8 @@ static Field convertToType(const Field & src, const IDataType & type) bool is_int64 = false; bool is_float32 = false; bool is_float64 = false; + bool is_date = false; + bool is_datetime = false; false || (is_uint8 = typeid_cast(&type)) @@ -257,7 +261,10 @@ static Field convertToType(const Field & src, const IDataType & type) || (is_int32 = typeid_cast(&type)) || (is_int64 = typeid_cast(&type)) || (is_float32 = typeid_cast(&type)) - || (is_float64 = typeid_cast(&type)); + || (is_float64 = typeid_cast(&type)) + || (is_date = typeid_cast(&type)) + || (is_datetime = typeid_cast(&type)) + ; if (is_uint8 || is_uint16 || is_uint32 || is_uint64) { @@ -327,6 +334,14 @@ static Field convertToType(const Field & src, const IDataType & type) throw Exception("Type mismatch in IN section: " + type.getName() + " at left, " + Field::Types::toString(src.getType()) + " at right"); } + else if (is_date || is_datetime) + { + if (src.getType() != Field::Types::UInt64) + throw Exception("Type mismatch in IN section: " + type.getName() + " at left, " + + Field::Types::toString(src.getType()) + " at right"); + + return src; + } } else { diff --git a/dbms/tests/queries/0_stateless/00172_constexprs_in_set.reference b/dbms/tests/queries/0_stateless/00172_constexprs_in_set.reference new file mode 100644 index 00000000000..c06d3de5a56 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00172_constexprs_in_set.reference @@ -0,0 +1,6 @@ +14 3 +1 +1 +1 +1 +1 diff --git a/dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql b/dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql new file mode 100644 index 00000000000..3c438417053 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00172_constexprs_in_set.sql @@ -0,0 +1,6 @@ +SELECT sumIf(number, x), sum(x) FROM (SELECT number, number IN (0 + 1, 2 + 3, toUInt64(concat('8', ''))) AS x FROM system.numbers LIMIT 10); +SELECT toDate('2015-06-12') IN toDate('2015-06-12'); +SELECT toDate('2015-06-12') IN (toDate('2015-06-12')); +SELECT today() IN (toDate('2014-01-01'), toDate(now())); +SELECT - -1 IN (2 - 1); +SELECT - -1 IN (2 - 1, 3); From 26813667eb8248614c3d77e7e73c28b15a9d4c59 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 12 Jun 2015 10:46:58 +0300 Subject: [PATCH 109/109] dbms: allowed to compare Date and DateTime with constant string [#METR-2944]. --- .../DB/Functions/FunctionsComparison.h | 214 +++++++++++++----- dbms/src/Interpreters/ExpressionActions.cpp | 3 +- ...e_date_time_with_constant_string.reference | 80 +++++++ ...compare_date_time_with_constant_string.sql | 83 +++++++ 4 files changed, 317 insertions(+), 63 deletions(-) create mode 100644 dbms/tests/queries/0_stateless/00173_compare_date_time_with_constant_string.reference create mode 100644 dbms/tests/queries/0_stateless/00173_compare_date_time_with_constant_string.sql diff --git a/dbms/include/DB/Functions/FunctionsComparison.h b/dbms/include/DB/Functions/FunctionsComparison.h index d459055367b..a901f363342 100644 --- a/dbms/include/DB/Functions/FunctionsComparison.h +++ b/dbms/include/DB/Functions/FunctionsComparison.h @@ -26,6 +26,10 @@ namespace DB * - даты; * - даты-с-временем; * внутри каждой группы, но не из разных групп. + * + * Исключение: можно сравнивать дату и дату-с-временем с константной строкой. Пример: EventDate = '2015-01-01'. + * + * TODO Массивы, кортежи. */ /** Игнорируем warning о сравнении signed и unsigned. @@ -391,9 +395,9 @@ public: private: template - bool executeNumRightType(Block & block, const ColumnNumbers & arguments, size_t result, const ColumnVector * col_left) + bool executeNumRightType(Block & block, size_t result, const ColumnVector * col_left, const IColumn * col_right_untyped) { - if (ColumnVector * col_right = typeid_cast *>(&*block.getByPosition(arguments[1]).column)) + if (const ColumnVector * col_right = typeid_cast *>(col_right_untyped)) { ColumnUInt8 * col_res = new ColumnUInt8; block.getByPosition(result).column = col_res; @@ -404,7 +408,7 @@ private: return true; } - else if (ColumnConst * col_right = typeid_cast *>(&*block.getByPosition(arguments[1]).column)) + else if (const ColumnConst * col_right = typeid_cast *>(col_right_untyped)) { ColumnUInt8 * col_res = new ColumnUInt8; block.getByPosition(result).column = col_res; @@ -420,9 +424,9 @@ private: } template - bool executeNumConstRightType(Block & block, const ColumnNumbers & arguments, size_t result, const ColumnConst * col_left) + bool executeNumConstRightType(Block & block, size_t result, const ColumnConst * col_left, const IColumn * col_right_untyped) { - if (ColumnVector * col_right = typeid_cast *>(&*block.getByPosition(arguments[1]).column)) + if (const ColumnVector * col_right = typeid_cast *>(col_right_untyped)) { ColumnUInt8 * col_res = new ColumnUInt8; block.getByPosition(result).column = col_res; @@ -433,7 +437,7 @@ private: return true; } - else if (ColumnConst * col_right = typeid_cast *>(&*block.getByPosition(arguments[1]).column)) + else if (const ColumnConst * col_right = typeid_cast *>(col_right_untyped)) { UInt8 res = 0; NumComparisonImpl>::constant_constant(col_left->getData(), col_right->getData(), res); @@ -448,41 +452,41 @@ private: } template - bool executeNumLeftType(Block & block, const ColumnNumbers & arguments, size_t result) + bool executeNumLeftType(Block & block, size_t result, const IColumn * col_left_untyped, const IColumn * col_right_untyped) { - if (ColumnVector * col_left = typeid_cast *>(&*block.getByPosition(arguments[0]).column)) + if (const ColumnVector * col_left = typeid_cast *>(col_left_untyped)) { - if ( executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left) - || executeNumRightType(block, arguments, result, col_left)) + if ( executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped) + || executeNumRightType(block, result, col_left, col_right_untyped)) return true; else - throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName() + throw Exception("Illegal column " + col_right_untyped->getName() + " of second argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } - else if (ColumnConst * col_left = typeid_cast *>(&*block.getByPosition(arguments[0]).column)) + else if (const ColumnConst * col_left = typeid_cast *>(col_left_untyped)) { - if ( executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left) - || executeNumConstRightType(block, arguments, result, col_left)) + if ( executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped) + || executeNumConstRightType(block, result, col_left, col_right_untyped)) return true; else - throw Exception("Illegal column " + block.getByPosition(arguments[1]).column->getName() + throw Exception("Illegal column " + col_right_untyped->getName() + " of second argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } @@ -490,17 +494,14 @@ private: return false; } - void executeString(Block & block, const ColumnNumbers & arguments, size_t result) + void executeString(Block & block, size_t result, const IColumn * c0, const IColumn * c1) { - IColumn * c0 = &*block.getByPosition(arguments[0]).column; - IColumn * c1 = &*block.getByPosition(arguments[1]).column; - - ColumnString * c0_string = typeid_cast(c0); - ColumnString * c1_string = typeid_cast(c1); - ColumnFixedString * c0_fixed_string = typeid_cast(c0); - ColumnFixedString * c1_fixed_string = typeid_cast(c1); - ColumnConstString * c0_const = typeid_cast(c0); - ColumnConstString * c1_const = typeid_cast(c1); + const ColumnString * c0_string = typeid_cast(c0); + const ColumnString * c1_string = typeid_cast(c1); + const ColumnFixedString * c0_fixed_string = typeid_cast(c0); + const ColumnFixedString * c1_fixed_string = typeid_cast(c1); + const ColumnConstString * c0_const = typeid_cast(c0); + const ColumnConstString * c1_const = typeid_cast(c1); using StringImpl = StringComparisonImpl>; @@ -559,13 +560,66 @@ private: c_res->getData()); else throw Exception("Illegal columns " - + block.getByPosition(arguments[0]).column->getName() + " and " - + block.getByPosition(arguments[1]).column->getName() + + c0->getName() + " and " + c1->getName() + " of arguments of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } } + void executeDateOrDateTimeWithConstString(Block & block, size_t result, + const IColumn * col_left_untyped, const IColumn * col_right_untyped, + bool left_is_num, bool right_is_num) + { + /// Особый случай - сравнение дат и дат-с-временем со строковой константой. + const IColumn * column_date_or_datetime = left_is_num ? col_left_untyped : col_right_untyped; + const IColumn * column_string_untyped = !left_is_num ? col_left_untyped : col_right_untyped; + + bool is_date = false; + bool is_date_time = false; + + is_date = typeid_cast *>(column_date_or_datetime) + || typeid_cast *>(column_date_or_datetime); + + if (!is_date) + is_date_time = typeid_cast *>(column_date_or_datetime) + || typeid_cast *>(column_date_or_datetime); + + const ColumnConstString * column_string = typeid_cast(column_string_untyped); + + if (!column_string + || (!is_date && !is_date_time)) + throw Exception("Illegal columns " + col_left_untyped->getName() + " and " + col_right_untyped->getName() + + " of arguments of function " + getName(), + ErrorCodes::ILLEGAL_COLUMN); + + if (is_date) + { + DayNum_t date; + ReadBufferFromString in(column_string->getData()); + readDateText(date, in); + if (!in.eof()) + throw Exception("String is too long for Date: " + column_string->getData()); + + ColumnConst parsed_const_date(block.rowsInFirstColumn(), date); + executeNumLeftType(block, result, + left_is_num ? col_left_untyped : &parsed_const_date, + left_is_num ? &parsed_const_date : col_right_untyped); + } + else if (is_date_time) + { + time_t date_time; + ReadBufferFromString in(column_string->getData()); + readDateTimeText(date_time, in); + if (!in.eof()) + throw Exception("String is too long for DateTime: " + column_string->getData()); + + ColumnConst parsed_const_date_time(block.rowsInFirstColumn(), date_time); + executeNumLeftType(block, result, + left_is_num ? col_left_untyped : &parsed_const_date_time, + left_is_num ? &parsed_const_date_time : col_right_untyped); + } + } + public: /// Получить имя функции. String getName() const @@ -581,12 +635,36 @@ public: + toString(arguments.size()) + ", should be 2.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - if (!( ( arguments[0]->isNumeric() && arguments[0]->behavesAsNumber() - && arguments[1]->isNumeric() && arguments[1]->behavesAsNumber()) - || ( (arguments[0]->getName() == "String" || arguments[0]->getName().substr(0, 11) == "FixedString") - && (arguments[1]->getName() == "String" || arguments[1]->getName().substr(0, 11) == "FixedString")) - || (arguments[0]->getName() == "Date" && arguments[1]->getName() == "Date") - || (arguments[0]->getName() == "DateTime" && arguments[1]->getName() == "DateTime"))) + bool left_is_date = false; + bool left_is_date_time = false; + bool left_is_string = false; + bool left_is_fixed_string = false; + + false + || (left_is_date = typeid_cast(arguments[0].get())) + || (left_is_date_time = typeid_cast(arguments[0].get())) + || (left_is_string = typeid_cast(arguments[0].get())) + || (left_is_fixed_string = typeid_cast(arguments[0].get())); + + bool right_is_date = false; + bool right_is_date_time = false; + bool right_is_string = false; + bool right_is_fixed_string = false; + + false + || (right_is_date = typeid_cast(arguments[1].get())) + || (right_is_date_time = typeid_cast(arguments[1].get())) + || (right_is_string = typeid_cast(arguments[1].get())) + || (right_is_fixed_string = typeid_cast(arguments[1].get())); + + if (!( (arguments[0]->behavesAsNumber() && arguments[1]->behavesAsNumber()) + || ((left_is_string || left_is_fixed_string) && (right_is_string || right_is_fixed_string)) + || (left_is_date && right_is_date) + || (left_is_date && right_is_string) /// Можно сравнивать дату и дату-с-временем с константной строкой. + || (left_is_string && right_is_date) + || (left_is_date_time && right_is_date_time) + || (left_is_date_time && right_is_string) + || (left_is_string && right_is_date_time))) throw Exception("Illegal types of arguments (" + arguments[0]->getName() + ", " + arguments[1]->getName() + ")" " of function " + getName(), ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -596,24 +674,36 @@ public: /// Выполнить функцию над блоком. void execute(Block & block, const ColumnNumbers & arguments, size_t result) { - if (block.getByPosition(arguments[0]).column->isNumeric()) + const IColumn * col_left_untyped = block.getByPosition(arguments[0]).column.get(); + const IColumn * col_right_untyped = block.getByPosition(arguments[1]).column.get(); + + bool left_is_num = col_left_untyped->isNumeric(); + bool right_is_num = col_right_untyped->isNumeric(); + + if (left_is_num && right_is_num) { - if (!( executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result) - || executeNumLeftType(block, arguments, result))) - throw Exception("Illegal column " + block.getByPosition(arguments[0]).column->getName() + if (!( executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped) + || executeNumLeftType(block, result, col_left_untyped, col_right_untyped))) + throw Exception("Illegal column " + col_left_untyped->getName() + " of first argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); } + else if (!left_is_num && !right_is_num) + { + executeString(block, result, col_left_untyped, col_right_untyped); + } else - executeString(block, arguments, result); + { + executeDateOrDateTimeWithConstString(block, result, col_left_untyped, col_right_untyped, left_is_num, right_is_num); + } } }; diff --git a/dbms/src/Interpreters/ExpressionActions.cpp b/dbms/src/Interpreters/ExpressionActions.cpp index fbfdd349381..c6b78cfbc5a 100644 --- a/dbms/src/Interpreters/ExpressionActions.cpp +++ b/dbms/src/Interpreters/ExpressionActions.cpp @@ -124,12 +124,13 @@ void ExpressionAction::prepare(Block & sample_block) /// Если все аргументы и требуемые столбцы - константы, выполним функцию. if (all_const) { + size_t result_position = sample_block.columns(); + ColumnWithNameAndType new_column; new_column.name = result_name; new_column.type = result_type; sample_block.insert(new_column); - size_t result_position = sample_block.getPositionByName(result_name); function->execute(sample_block, arguments, prerequisites, result_position); /// Если получилась не константа, на всякий случай будем считать результат неизвестным. diff --git a/dbms/tests/queries/0_stateless/00173_compare_date_time_with_constant_string.reference b/dbms/tests/queries/0_stateless/00173_compare_date_time_with_constant_string.reference new file mode 100644 index 00000000000..7f8dca8f200 --- /dev/null +++ b/dbms/tests/queries/0_stateless/00173_compare_date_time_with_constant_string.reference @@ -0,0 +1,80 @@ +1 +1 +0 +0 +1 +1 +0 +0 +1 +1 +0 +0 +0 +0 +1 +1 +0 +0 +1 +1 +1 +1 +0 +0 +1 +1 +0 +0 +1 +1 +0 +0 +0 +0 +1 +1 +0 +0 +1 +1 +1 +1 +0 +0 +1 +1 +0 +0 +1 +1 +0 +0 +0 +0 +1 +1 +0 +0 +1 +1 +1 +1 +0 +0 +1 +1 +0 +0 +1 +1 +0 +0 +0 +0 +1 +1 +0 +0 +1 +1 diff --git a/dbms/tests/queries/0_stateless/00173_compare_date_time_with_constant_string.sql b/dbms/tests/queries/0_stateless/00173_compare_date_time_with_constant_string.sql new file mode 100644 index 00000000000..c89292a252a --- /dev/null +++ b/dbms/tests/queries/0_stateless/00173_compare_date_time_with_constant_string.sql @@ -0,0 +1,83 @@ +SELECT toDate('2015-02-03') = '2015-02-03'; +SELECT '2015-02-03' = toDate('2015-02-03'); +SELECT toDate('2015-02-03') = '2015-02-04'; +SELECT '2015-02-03' = toDate('2015-02-04'); +SELECT toDate('2015-02-03') < '2015-02-04'; +SELECT '2015-02-03' < toDate('2015-02-04'); +SELECT toDate('2015-02-03') > '2015-02-04'; +SELECT '2015-02-03' > toDate('2015-02-04'); +SELECT toDate('2015-02-03') <= '2015-02-04'; +SELECT '2015-02-03' <= toDate('2015-02-04'); +SELECT toDate('2015-02-03') >= '2015-02-04'; +SELECT '2015-02-03' >= toDate('2015-02-04'); +SELECT toDate('2015-02-05') < '2015-02-04'; +SELECT '2015-02-05' < toDate('2015-02-04'); +SELECT toDate('2015-02-05') > '2015-02-04'; +SELECT '2015-02-05' > toDate('2015-02-04'); +SELECT toDate('2015-02-05') <= '2015-02-04'; +SELECT '2015-02-05' <= toDate('2015-02-04'); +SELECT toDate('2015-02-05') >= '2015-02-04'; +SELECT '2015-02-05' >= toDate('2015-02-04'); + +SELECT materialize(toDate('2015-02-03')) = '2015-02-03'; +SELECT '2015-02-03' = materialize(toDate('2015-02-03')); +SELECT materialize(toDate('2015-02-03')) = '2015-02-04'; +SELECT '2015-02-03' = materialize(toDate('2015-02-04')); +SELECT materialize(toDate('2015-02-03')) < '2015-02-04'; +SELECT '2015-02-03' < materialize(toDate('2015-02-04')); +SELECT materialize(toDate('2015-02-03')) > '2015-02-04'; +SELECT '2015-02-03' > materialize(toDate('2015-02-04')); +SELECT materialize(toDate('2015-02-03')) <= '2015-02-04'; +SELECT '2015-02-03' <= materialize(toDate('2015-02-04')); +SELECT materialize(toDate('2015-02-03')) >= '2015-02-04'; +SELECT '2015-02-03' >= materialize(toDate('2015-02-04')); +SELECT materialize(toDate('2015-02-05')) < '2015-02-04'; +SELECT '2015-02-05' < materialize(toDate('2015-02-04')); +SELECT materialize(toDate('2015-02-05')) > '2015-02-04'; +SELECT '2015-02-05' > materialize(toDate('2015-02-04')); +SELECT materialize(toDate('2015-02-05')) <= '2015-02-04'; +SELECT '2015-02-05' <= materialize(toDate('2015-02-04')); +SELECT materialize(toDate('2015-02-05')) >= '2015-02-04'; +SELECT '2015-02-05' >= materialize(toDate('2015-02-04')); + +SELECT toDateTime('2015-02-03 04:05:06') = '2015-02-03 04:05:06'; +SELECT '2015-02-03 04:05:06' = toDateTime('2015-02-03 04:05:06'); +SELECT toDateTime('2015-02-03 04:05:06') = '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' = toDateTime('2015-02-03 05:06:07'); +SELECT toDateTime('2015-02-03 04:05:06') < '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' < toDateTime('2015-02-03 05:06:07'); +SELECT toDateTime('2015-02-03 04:05:06') > '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' > toDateTime('2015-02-03 05:06:07'); +SELECT toDateTime('2015-02-03 04:05:06') <= '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' <= toDateTime('2015-02-03 05:06:07'); +SELECT toDateTime('2015-02-03 04:05:06') >= '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' >= toDateTime('2015-02-03 05:06:07'); +SELECT toDateTime('2015-02-03 06:07:08') < '2015-02-03 05:06:07'; +SELECT '2015-02-03 06:07:08' < toDateTime('2015-02-03 05:06:07'); +SELECT toDateTime('2015-02-03 06:07:08') > '2015-02-03 05:06:07'; +SELECT '2015-02-03 06:07:08' > toDateTime('2015-02-03 05:06:07'); +SELECT toDateTime('2015-02-03 06:07:08') <= '2015-02-03 05:06:07'; +SELECT '2015-02-03 06:07:08' <= toDateTime('2015-02-03 05:06:07'); +SELECT toDateTime('2015-02-03 06:07:08') >= '2015-02-03 05:06:07'; +SELECT '2015-02-03 06:07:08' >= toDateTime('2015-02-03 05:06:07'); + +SELECT materialize(toDateTime('2015-02-03 04:05:06')) = '2015-02-03 04:05:06'; +SELECT '2015-02-03 04:05:06' = materialize(toDateTime('2015-02-03 04:05:06')); +SELECT materialize(toDateTime('2015-02-03 04:05:06')) = '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' = materialize(toDateTime('2015-02-03 05:06:07')); +SELECT materialize(toDateTime('2015-02-03 04:05:06')) < '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' < materialize(toDateTime('2015-02-03 05:06:07')); +SELECT materialize(toDateTime('2015-02-03 04:05:06')) > '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' > materialize(toDateTime('2015-02-03 05:06:07')); +SELECT materialize(toDateTime('2015-02-03 04:05:06')) <= '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' <= materialize(toDateTime('2015-02-03 05:06:07')); +SELECT materialize(toDateTime('2015-02-03 04:05:06')) >= '2015-02-03 05:06:07'; +SELECT '2015-02-03 04:05:06' >= materialize(toDateTime('2015-02-03 05:06:07')); +SELECT materialize(toDateTime('2015-02-03 06:07:08')) < '2015-02-03 05:06:07'; +SELECT '2015-02-03 06:07:08' < materialize(toDateTime('2015-02-03 05:06:07')); +SELECT materialize(toDateTime('2015-02-03 06:07:08')) > '2015-02-03 05:06:07'; +SELECT '2015-02-03 06:07:08' > materialize(toDateTime('2015-02-03 05:06:07')); +SELECT materialize(toDateTime('2015-02-03 06:07:08')) <= '2015-02-03 05:06:07'; +SELECT '2015-02-03 06:07:08' <= materialize(toDateTime('2015-02-03 05:06:07')); +SELECT materialize(toDateTime('2015-02-03 06:07:08')) >= '2015-02-03 05:06:07'; +SELECT '2015-02-03 06:07:08' >= materialize(toDateTime('2015-02-03 05:06:07'));