From 5a468a5d324965053b50b51cfefe5009b324de03 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 4 Feb 2021 10:05:28 -0800 Subject: [PATCH 001/220] ServerUUID - initial implementation --- programs/server/Server.cpp | 8 +++ src/Common/ServerUUIDFile.cpp | 100 ++++++++++++++++++++++++++++++++++ src/Common/ServerUUIDFile.h | 34 ++++++++++++ src/Common/ya.make | 1 + 4 files changed, 143 insertions(+) create mode 100644 src/Common/ServerUUIDFile.cpp create mode 100644 src/Common/ServerUUIDFile.h diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index a96cb2b8973..6037bdc8ce0 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -61,6 +61,7 @@ #include #include "MetricsTransmitter.h" #include +#include #include #include #include @@ -563,6 +564,7 @@ int Server::main(const std::vector & /*args*/) global_context->setPath(path); StatusFile status{path + "status", StatusFile::write_full_info}; + ServerUUIDFile uuid{path + "server_uuid", ServerUUIDFile::write_server_uuid}; /// Try to increase limit on number of open files. { @@ -603,6 +605,12 @@ int Server::main(const std::vector & /*args*/) setupTmpPath(log, disk->getPath()); } + /// write unique server UUID + { + Poco::File(path + "uuidfile").createFile(); + + } + /** Directory with 'flags': files indicating temporary settings for the server set by system administrator. * Flags may be cleared automatically after being applied by the server. * Examples: do repair of local data; clone all replicated tables from replica. diff --git a/src/Common/ServerUUIDFile.cpp b/src/Common/ServerUUIDFile.cpp new file mode 100644 index 00000000000..76dc3996dd4 --- /dev/null +++ b/src/Common/ServerUUIDFile.cpp @@ -0,0 +1,100 @@ +#include "ServerUUIDFile.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ +extern const int CANNOT_OPEN_FILE; +extern const int CANNOT_CLOSE_FILE; +extern const int CANNOT_TRUNCATE_FILE; +extern const int CANNOT_SEEK_THROUGH_FILE; +} + + +ServerUUIDFile::FillFunction ServerUUIDFile::write_server_uuid = [](WriteBuffer & out) +{ + // TODO: compute random uuid + out << "736833cf-2224-475b-82e2-cbc114407345"; +}; + + +ServerUUIDFile::ServerUUIDFile(std::string path_, FillFunction fill_) + : path(std::move(path_)), fill(std::move(fill_)) +{ + /// If file already exists. NOTE Minor race condition. + if (Poco::File(path).exists()) + { + std::string contents; + { + ReadBufferFromFile in(path, 1024); + LimitReadBuffer limit_in(in, 1024, false); + readStringUntilEOF(contents, limit_in); + } + + if (!contents.empty()) + LOG_INFO(&Poco::Logger::get("ServerUUIDFile"), "Server UUID file {} already exists - unclean restart. Contents:\n{}", path, contents); + else + LOG_INFO(&Poco::Logger::get("ServerUUIDFile"), "Server UUID file {} already exists and is empty - probably unclean hardware restart.", path); + } + + fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); + + if (-1 == fd) + throwFromErrnoWithPath("Cannot open file " + path, path, ErrorCodes::CANNOT_OPEN_FILE); + + try + { + int flock_ret = flock(fd, LOCK_EX | LOCK_NB); + if (-1 == flock_ret) + { + if (errno == EWOULDBLOCK) + throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE); + else + throwFromErrnoWithPath("Cannot lock file " + path, path, ErrorCodes::CANNOT_OPEN_FILE); + } + + if (0 != ftruncate(fd, 0)) + throwFromErrnoWithPath("Cannot ftruncate " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE); + + if (0 != lseek(fd, 0, SEEK_SET)) + throwFromErrnoWithPath("Cannot lseek " + path, path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE); + + /// Write information about current server instance to the file. + WriteBufferFromFileDescriptor out(fd, 1024); + fill(out); + } + catch (...) + { + close(fd); + throw; + } +} + + +ServerUUIDFile::~ServerUUIDFile() +{ + if (0 != close(fd)) + LOG_ERROR(&Poco::Logger::get("ServerUUIDFile"), "Cannot close file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); + + if (0 != unlink(path.c_str())) + LOG_ERROR(&Poco::Logger::get("ServerUUIDFile"), "Cannot unlink file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); +} + +} diff --git a/src/Common/ServerUUIDFile.h b/src/Common/ServerUUIDFile.h new file mode 100644 index 00000000000..1783527d75a --- /dev/null +++ b/src/Common/ServerUUIDFile.h @@ -0,0 +1,34 @@ +#pragma once + +#include +#include +#include + + +namespace DB +{ + +class WriteBuffer; + + +/** Provides that no more than one server works with one data directory. + */ +class ServerUUIDFile : private boost::noncopyable +{ +public: + using FillFunction = std::function; + + ServerUUIDFile(std::string path_, FillFunction fill_); + ~ServerUUIDFile(); + + /// You can use one of these functions to fill the file or provide your own. + static FillFunction write_server_uuid; + +private: + const std::string path; + FillFunction fill; + int fd = -1; +}; + + +} diff --git a/src/Common/ya.make b/src/Common/ya.make index 64dd628c457..372f635ae14 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -59,6 +59,7 @@ SRCS( RWLock.cpp RemoteHostFilter.cpp SensitiveDataMasker.cpp + ServerUUIDFile.cpp SettingsChanges.cpp SharedLibrary.cpp ShellCommand.cpp From 14d2d68cf79e58235475d1ca08453a17299f0c40 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 4 Feb 2021 11:34:18 -0800 Subject: [PATCH 002/220] ServerUUID - generate random uuid --- src/Common/ServerUUIDFile.cpp | 31 +++++++++++++++++++++++++------ src/Common/ServerUUIDFile.h | 5 ++--- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/Common/ServerUUIDFile.cpp b/src/Common/ServerUUIDFile.cpp index 76dc3996dd4..92fc94e34d9 100644 --- a/src/Common/ServerUUIDFile.cpp +++ b/src/Common/ServerUUIDFile.cpp @@ -5,10 +5,9 @@ #include #include +#include #include #include -#include -#include #include #include @@ -28,10 +27,30 @@ extern const int CANNOT_SEEK_THROUGH_FILE; } -ServerUUIDFile::FillFunction ServerUUIDFile::write_server_uuid = [](WriteBuffer & out) -{ - // TODO: compute random uuid - out << "736833cf-2224-475b-82e2-cbc114407345"; +ServerUUIDFile::FillFunction ServerUUIDFile::write_server_uuid = [](WriteBuffer & out) { + union + { + char bytes[16]; + struct + { + UInt64 a; + UInt64 b; + } words; + __uint128_t uuid; + } random; + + random.words.a = thread_local_rng(); //-V656 + random.words.b = thread_local_rng(); //-V656 + + struct QueryUUID : Poco::UUID + { + QueryUUID(const char * bytes, Poco::UUID::Version version) + : Poco::UUID(bytes, version) {} + }; + + auto server_uuid = QueryUUID(random.bytes, Poco::UUID::UUID_RANDOM).toString(); + + out << server_uuid; }; diff --git a/src/Common/ServerUUIDFile.h b/src/Common/ServerUUIDFile.h index 1783527d75a..b85ce91d8a2 100644 --- a/src/Common/ServerUUIDFile.h +++ b/src/Common/ServerUUIDFile.h @@ -1,13 +1,12 @@ #pragma once -#include #include +#include #include namespace DB { - class WriteBuffer; @@ -16,7 +15,7 @@ class WriteBuffer; class ServerUUIDFile : private boost::noncopyable { public: - using FillFunction = std::function; + using FillFunction = std::function; ServerUUIDFile(std::string path_, FillFunction fill_); ~ServerUUIDFile(); From daf46d21d8787d4b7d230a92b48e97e4763bb783 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 4 Feb 2021 11:54:55 -0800 Subject: [PATCH 003/220] ServerUUID - fix writing uuid file --- programs/server/Server.cpp | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 6037bdc8ce0..599083ed320 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -564,7 +564,7 @@ int Server::main(const std::vector & /*args*/) global_context->setPath(path); StatusFile status{path + "status", StatusFile::write_full_info}; - ServerUUIDFile uuid{path + "server_uuid", ServerUUIDFile::write_server_uuid}; + ServerUUIDFile uuid{path + "uuid", ServerUUIDFile::write_server_uuid}; /// Try to increase limit on number of open files. { @@ -605,12 +605,6 @@ int Server::main(const std::vector & /*args*/) setupTmpPath(log, disk->getPath()); } - /// write unique server UUID - { - Poco::File(path + "uuidfile").createFile(); - - } - /** Directory with 'flags': files indicating temporary settings for the server set by system administrator. * Flags may be cleared automatically after being applied by the server. * Examples: do repair of local data; clone all replicated tables from replica. From 6624dfb7eaf5b28a8974566e0274a408ecae2410 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 4 Feb 2021 18:21:52 -0800 Subject: [PATCH 004/220] ServerUUID - fix naming --- src/Common/ServerUUIDFile.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Common/ServerUUIDFile.cpp b/src/Common/ServerUUIDFile.cpp index 92fc94e34d9..bf094d39fdd 100644 --- a/src/Common/ServerUUIDFile.cpp +++ b/src/Common/ServerUUIDFile.cpp @@ -42,13 +42,12 @@ ServerUUIDFile::FillFunction ServerUUIDFile::write_server_uuid = [](WriteBuffer random.words.a = thread_local_rng(); //-V656 random.words.b = thread_local_rng(); //-V656 - struct QueryUUID : Poco::UUID + struct ServerUUID : Poco::UUID { - QueryUUID(const char * bytes, Poco::UUID::Version version) - : Poco::UUID(bytes, version) {} + ServerUUID(const char * bytes, Poco::UUID::Version version) : Poco::UUID(bytes, version) { } }; - auto server_uuid = QueryUUID(random.bytes, Poco::UUID::UUID_RANDOM).toString(); + auto server_uuid = ServerUUID(random.bytes, Poco::UUID::UUID_RANDOM).toString(); out << server_uuid; }; From 717ff0579713a7433232d497d4d301212b2303b4 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 4 Feb 2021 18:44:37 -0800 Subject: [PATCH 005/220] ServerUUID - write uuid file for LocalServer --- programs/local/LocalServer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 5a8d35e204d..1cf369614ea 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -217,6 +218,7 @@ try tryInitPath(); std::optional status; + std::optional server_uuid; /// Skip temp path installation @@ -279,6 +281,7 @@ try /// Lock path directory before read status.emplace(path + "status", StatusFile::write_full_info); + server_uuid.emplace(path + "uuid", ServerUUIDFile::write_server_uuid); LOG_DEBUG(log, "Loading metadata from {}", path); Poco::File(path + "data/").createDirectories(); From 69d16059745373157ceab890d579678882da0942 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 4 Feb 2021 20:09:08 -0800 Subject: [PATCH 006/220] ServerUUID - fix formatting and style checks --- src/Common/ServerUUIDFile.cpp | 40 +++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/src/Common/ServerUUIDFile.cpp b/src/Common/ServerUUIDFile.cpp index bf094d39fdd..d1627e47b63 100644 --- a/src/Common/ServerUUIDFile.cpp +++ b/src/Common/ServerUUIDFile.cpp @@ -1,33 +1,33 @@ #include "ServerUUIDFile.h" -#include -#include #include +#include +#include #include #include -#include #include +#include -#include #include -#include #include +#include +#include namespace DB { - namespace ErrorCodes { -extern const int CANNOT_OPEN_FILE; -extern const int CANNOT_CLOSE_FILE; -extern const int CANNOT_TRUNCATE_FILE; -extern const int CANNOT_SEEK_THROUGH_FILE; + extern const int CANNOT_OPEN_FILE; + extern const int CANNOT_CLOSE_FILE; + extern const int CANNOT_TRUNCATE_FILE; + extern const int CANNOT_SEEK_THROUGH_FILE; } -ServerUUIDFile::FillFunction ServerUUIDFile::write_server_uuid = [](WriteBuffer & out) { +ServerUUIDFile::FillFunction ServerUUIDFile::write_server_uuid = [](WriteBuffer & out) +{ union { char bytes[16]; @@ -53,8 +53,7 @@ ServerUUIDFile::FillFunction ServerUUIDFile::write_server_uuid = [](WriteBuffer }; -ServerUUIDFile::ServerUUIDFile(std::string path_, FillFunction fill_) - : path(std::move(path_)), fill(std::move(fill_)) +ServerUUIDFile::ServerUUIDFile(std::string path_, FillFunction fill_) : path(std::move(path_)), fill(std::move(fill_)) { /// If file already exists. NOTE Minor race condition. if (Poco::File(path).exists()) @@ -67,9 +66,16 @@ ServerUUIDFile::ServerUUIDFile(std::string path_, FillFunction fill_) } if (!contents.empty()) - LOG_INFO(&Poco::Logger::get("ServerUUIDFile"), "Server UUID file {} already exists - unclean restart. Contents:\n{}", path, contents); + LOG_INFO( + &Poco::Logger::get("ServerUUIDFile"), + "Server UUID file {} already exists - unclean restart. Contents:\n{}", + path, + contents); else - LOG_INFO(&Poco::Logger::get("ServerUUIDFile"), "Server UUID file {} already exists and is empty - probably unclean hardware restart.", path); + LOG_INFO( + &Poco::Logger::get("ServerUUIDFile"), + "Server UUID file {} already exists and is empty - probably unclean hardware restart.", + path); } fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); @@ -83,7 +89,9 @@ ServerUUIDFile::ServerUUIDFile(std::string path_, FillFunction fill_) if (-1 == flock_ret) { if (errno == EWOULDBLOCK) - throw Exception("Cannot lock file " + path + ". Another server instance in same directory is already running.", ErrorCodes::CANNOT_OPEN_FILE); + throw Exception( + "Cannot lock file " + path + ". Another server instance in same directory is already running.", + ErrorCodes::CANNOT_OPEN_FILE); else throwFromErrnoWithPath("Cannot lock file " + path, path, ErrorCodes::CANNOT_OPEN_FILE); } From a21ff1faf74c2f074b72586f69b421a6c60cbc4f Mon Sep 17 00:00:00 2001 From: bharatnc Date: Wed, 10 Feb 2021 18:14:12 -0800 Subject: [PATCH 007/220] ServerUUID - simplify UUID generation as per review --- programs/local/LocalServer.cpp | 3 - programs/server/Server.cpp | 133 ++++++++++++++++++++------------- src/Common/ServerUUIDFile.cpp | 126 ------------------------------- src/Common/ServerUUIDFile.h | 33 -------- src/Common/ya.make | 1 - 5 files changed, 80 insertions(+), 216 deletions(-) delete mode 100644 src/Common/ServerUUIDFile.cpp delete mode 100644 src/Common/ServerUUIDFile.h diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 1cf369614ea..5a8d35e204d 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -28,7 +28,6 @@ #include #include #include -#include #include #include #include @@ -218,7 +217,6 @@ try tryInitPath(); std::optional status; - std::optional server_uuid; /// Skip temp path installation @@ -281,7 +279,6 @@ try /// Lock path directory before read status.emplace(path + "status", StatusFile::write_full_info); - server_uuid.emplace(path + "uuid", ServerUUIDFile::write_server_uuid); LOG_DEBUG(log, "Loading metadata from {}", path); Poco::File(path + "data/").createDirectories(); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 599083ed320..7dde93d58fe 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1,62 +1,45 @@ #include "Server.h" #include -#include -#include -#include -#include #include #include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include #include #include #include #include +#include +#include #include +#include #include #include #include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include #include #include -#include -#include -#include -#include #include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include "MetricsTransmitter.h" @@ -64,20 +47,35 @@ #include #include #include +#include +#include +#include #include +#include +#include +#include +#include #include -#include -#include -#include -#include - +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "MetricsTransmitter.h" #if !defined(ARCADIA_BUILD) -# include "config_core.h" -# include "Common/config_version.h" -# if USE_OPENCL -# include "Common/BitonicSort.h" // Y_IGNORE -# endif +# include "Common/config_version.h" +# include "config_core.h" +# if USE_OPENCL +# include "Common/BitonicSort.h" // Y_IGNORE +# endif #endif #if defined(OS_LINUX) @@ -105,6 +103,7 @@ namespace CurrentMetrics extern const Metric MemoryTracking; } +namespace fs = std::filesystem; int mainEntryClickHouseServer(int argc, char ** argv) { @@ -564,7 +563,35 @@ int Server::main(const std::vector & /*args*/) global_context->setPath(path); StatusFile status{path + "status", StatusFile::write_full_info}; - ServerUUIDFile uuid{path + "uuid", ServerUUIDFile::write_server_uuid}; + + + /// Write a uuid file containing a unique uuid if the file doesn't already exist during server start. + { + fs::path server_uuid_file(path + "uuid"); + + if (!fs::exists(server_uuid_file)) + { + try + { + /// Note: Poco::UUIDGenerator().createRandom() uses /dev/random and can be expensive. But since + /// it's only going to be generated once (i.e if the uuid file doesn't exist), it's probably fine. + auto uuid_str = Poco::UUIDGenerator().createRandom().toString(); + WriteBufferFromFile out(server_uuid_file.string()); + out.write(uuid_str.data(), uuid_str.size()); + out.sync(); + out.finalize(); + } + catch (...) + { + throw Poco::Exception("Caught Exception while writing to write UUID file {}.\n", server_uuid_file.string()); + } + LOG_INFO(log, "Server UUID file {} containing a unique UUID has been written.\n", server_uuid_file.string()); + } + else + { + LOG_WARNING(log, "Server UUID file {} already exists, will keep it.\n", server_uuid_file.string()); + } + } /// Try to increase limit on number of open files. { diff --git a/src/Common/ServerUUIDFile.cpp b/src/Common/ServerUUIDFile.cpp deleted file mode 100644 index d1627e47b63..00000000000 --- a/src/Common/ServerUUIDFile.cpp +++ /dev/null @@ -1,126 +0,0 @@ -#include "ServerUUIDFile.h" - -#include -#include -#include - -#include -#include -#include -#include - -#include -#include -#include -#include - - -namespace DB -{ -namespace ErrorCodes -{ - extern const int CANNOT_OPEN_FILE; - extern const int CANNOT_CLOSE_FILE; - extern const int CANNOT_TRUNCATE_FILE; - extern const int CANNOT_SEEK_THROUGH_FILE; -} - - -ServerUUIDFile::FillFunction ServerUUIDFile::write_server_uuid = [](WriteBuffer & out) -{ - union - { - char bytes[16]; - struct - { - UInt64 a; - UInt64 b; - } words; - __uint128_t uuid; - } random; - - random.words.a = thread_local_rng(); //-V656 - random.words.b = thread_local_rng(); //-V656 - - struct ServerUUID : Poco::UUID - { - ServerUUID(const char * bytes, Poco::UUID::Version version) : Poco::UUID(bytes, version) { } - }; - - auto server_uuid = ServerUUID(random.bytes, Poco::UUID::UUID_RANDOM).toString(); - - out << server_uuid; -}; - - -ServerUUIDFile::ServerUUIDFile(std::string path_, FillFunction fill_) : path(std::move(path_)), fill(std::move(fill_)) -{ - /// If file already exists. NOTE Minor race condition. - if (Poco::File(path).exists()) - { - std::string contents; - { - ReadBufferFromFile in(path, 1024); - LimitReadBuffer limit_in(in, 1024, false); - readStringUntilEOF(contents, limit_in); - } - - if (!contents.empty()) - LOG_INFO( - &Poco::Logger::get("ServerUUIDFile"), - "Server UUID file {} already exists - unclean restart. Contents:\n{}", - path, - contents); - else - LOG_INFO( - &Poco::Logger::get("ServerUUIDFile"), - "Server UUID file {} already exists and is empty - probably unclean hardware restart.", - path); - } - - fd = ::open(path.c_str(), O_WRONLY | O_CREAT | O_CLOEXEC, 0666); - - if (-1 == fd) - throwFromErrnoWithPath("Cannot open file " + path, path, ErrorCodes::CANNOT_OPEN_FILE); - - try - { - int flock_ret = flock(fd, LOCK_EX | LOCK_NB); - if (-1 == flock_ret) - { - if (errno == EWOULDBLOCK) - throw Exception( - "Cannot lock file " + path + ". Another server instance in same directory is already running.", - ErrorCodes::CANNOT_OPEN_FILE); - else - throwFromErrnoWithPath("Cannot lock file " + path, path, ErrorCodes::CANNOT_OPEN_FILE); - } - - if (0 != ftruncate(fd, 0)) - throwFromErrnoWithPath("Cannot ftruncate " + path, path, ErrorCodes::CANNOT_TRUNCATE_FILE); - - if (0 != lseek(fd, 0, SEEK_SET)) - throwFromErrnoWithPath("Cannot lseek " + path, path, ErrorCodes::CANNOT_SEEK_THROUGH_FILE); - - /// Write information about current server instance to the file. - WriteBufferFromFileDescriptor out(fd, 1024); - fill(out); - } - catch (...) - { - close(fd); - throw; - } -} - - -ServerUUIDFile::~ServerUUIDFile() -{ - if (0 != close(fd)) - LOG_ERROR(&Poco::Logger::get("ServerUUIDFile"), "Cannot close file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); - - if (0 != unlink(path.c_str())) - LOG_ERROR(&Poco::Logger::get("ServerUUIDFile"), "Cannot unlink file {}, {}", path, errnoToString(ErrorCodes::CANNOT_CLOSE_FILE)); -} - -} diff --git a/src/Common/ServerUUIDFile.h b/src/Common/ServerUUIDFile.h deleted file mode 100644 index b85ce91d8a2..00000000000 --- a/src/Common/ServerUUIDFile.h +++ /dev/null @@ -1,33 +0,0 @@ -#pragma once - -#include -#include -#include - - -namespace DB -{ -class WriteBuffer; - - -/** Provides that no more than one server works with one data directory. - */ -class ServerUUIDFile : private boost::noncopyable -{ -public: - using FillFunction = std::function; - - ServerUUIDFile(std::string path_, FillFunction fill_); - ~ServerUUIDFile(); - - /// You can use one of these functions to fill the file or provide your own. - static FillFunction write_server_uuid; - -private: - const std::string path; - FillFunction fill; - int fd = -1; -}; - - -} diff --git a/src/Common/ya.make b/src/Common/ya.make index 372f635ae14..64dd628c457 100644 --- a/src/Common/ya.make +++ b/src/Common/ya.make @@ -59,7 +59,6 @@ SRCS( RWLock.cpp RemoteHostFilter.cpp SensitiveDataMasker.cpp - ServerUUIDFile.cpp SettingsChanges.cpp SharedLibrary.cpp ShellCommand.cpp From 0123911f8bcfcf9f2e2f718ab9fd024284cd4208 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sat, 13 Feb 2021 02:35:20 +0300 Subject: [PATCH 008/220] Update Server.cpp --- programs/server/Server.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7dde93d58fe..b09b17127f3 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -567,7 +567,7 @@ int Server::main(const std::vector & /*args*/) /// Write a uuid file containing a unique uuid if the file doesn't already exist during server start. { - fs::path server_uuid_file(path + "uuid"); + fs::path server_uuid_file = fs::path(path) / "uuid"; if (!fs::exists(server_uuid_file)) { From 8c7f1e020412ba5e5e1a7f45902aa20f08453557 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 12 Feb 2021 15:51:14 -0800 Subject: [PATCH 009/220] Change logging to info and preserve exception --- programs/server/Server.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b09b17127f3..3c0dd98b7ce 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -583,13 +583,15 @@ int Server::main(const std::vector & /*args*/) } catch (...) { - throw Poco::Exception("Caught Exception while writing to write UUID file {}.\n", server_uuid_file.string()); + std::string message + = "Caught Exception " + getCurrentExceptionMessage(false) + " writing to write UUID file " + server_uuid_file.string(); + throw Poco::Exception(message); } LOG_INFO(log, "Server UUID file {} containing a unique UUID has been written.\n", server_uuid_file.string()); } else { - LOG_WARNING(log, "Server UUID file {} already exists, will keep it.\n", server_uuid_file.string()); + LOG_INFO(log, "Server UUID file {} already exists, will keep it.\n", server_uuid_file.string()); } } From 2f3fca352910936055e981268bb786e427377579 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Fri, 12 Feb 2021 16:42:40 -0800 Subject: [PATCH 010/220] change exception message slightly --- programs/server/Server.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 3c0dd98b7ce..27e3d523097 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -583,9 +583,9 @@ int Server::main(const std::vector & /*args*/) } catch (...) { - std::string message - = "Caught Exception " + getCurrentExceptionMessage(false) + " writing to write UUID file " + server_uuid_file.string(); - throw Poco::Exception(message); + throw Poco::Exception( + "Caught Exception " + getCurrentExceptionMessage(false) + " while writing the Server UUID file " + + server_uuid_file.string()); } LOG_INFO(log, "Server UUID file {} containing a unique UUID has been written.\n", server_uuid_file.string()); } From 0a3d16196a7a7b27794f8a02cd639e22b72e8d0b Mon Sep 17 00:00:00 2001 From: bharatnc Date: Sat, 13 Feb 2021 21:50:48 -0800 Subject: [PATCH 011/220] fix rebase issues --- programs/server/Server.cpp | 103 +++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 27e3d523097..979da949bbe 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -1,81 +1,82 @@ #include "Server.h" #include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include #include #include #include -#include -#include #include -#include #include #include #include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include #include #include -#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include #include #include #include "MetricsTransmitter.h" #include -#include #include #include -#include -#include -#include #include -#include -#include -#include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include "MetricsTransmitter.h" +#include +#include +#include +#include #if !defined(ARCADIA_BUILD) -# include "Common/config_version.h" -# include "config_core.h" -# if USE_OPENCL -# include "Common/BitonicSort.h" // Y_IGNORE -# endif +# include "config_core.h" +# include "Common/config_version.h" +# if USE_OPENCL +# include "Common/BitonicSort.h" // Y_IGNORE +# endif #endif #if defined(OS_LINUX) From 03bf6c540f1029aaaa2ca6005d7482f4cd304587 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 21 Jul 2021 04:41:55 +0300 Subject: [PATCH 012/220] Do not allow to create columns in block with identical name and different structure --- src/Core/Block.cpp | 152 ++++++++++++++++++++++++++------------------- 1 file changed, 89 insertions(+), 63 deletions(-) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index efd8de43a3c..6f106aa06f6 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -22,6 +22,85 @@ namespace ErrorCodes extern const int POSITION_OUT_OF_BOUND; extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; + extern const int BAD_ARGUMENTS; +} + +template +static ReturnType onError(const std::string & message [[maybe_unused]], int code [[maybe_unused]]) +{ + if constexpr (std::is_same_v) + throw Exception(message, code); + else + return false; +}; + + +template +static ReturnType checkColumnStructure(const ColumnWithTypeAndName & actual, const ColumnWithTypeAndName & expected, + const std::string & context_description, bool allow_remove_constants, int code) +{ + if (actual.name != expected.name) + return onError("Block structure mismatch in " + context_description + " stream: different names of columns:\n" + + actual.dumpStructure() + "\n" + expected.dumpStructure(), code); + + if (!actual.type->equals(*expected.type)) + return onError("Block structure mismatch in " + context_description + " stream: different types:\n" + + actual.dumpStructure() + "\n" + expected.dumpStructure(), code); + + if (!actual.column || !expected.column) + return ReturnType(true); + + const IColumn * actual_column = actual.column.get(); + + /// If we allow to remove constants, and expected column is not const, then unwrap actual constant column. + if (allow_remove_constants && !isColumnConst(*expected.column)) + { + if (const auto * column_const = typeid_cast(actual_column)) + actual_column = &column_const->getDataColumn(); + } + + if (actual_column->getName() != expected.column->getName()) + return onError("Block structure mismatch in " + context_description + " stream: different columns:\n" + + actual.dumpStructure() + "\n" + expected.dumpStructure(), code); + + if (isColumnConst(*actual.column) && isColumnConst(*expected.column)) + { + Field actual_value = assert_cast(*actual.column).getField(); + Field expected_value = assert_cast(*expected.column).getField(); + + if (actual_value != expected_value) + return onError("Block structure mismatch in " + context_description + " stream: different values of constants, actual: " + + applyVisitor(FieldVisitorToString(), actual_value) + ", expected: " + applyVisitor(FieldVisitorToString(), expected_value), + code); + } + + return ReturnType(true); +} + + +template +static ReturnType checkBlockStructure(const Block & lhs, const Block & rhs, const std::string & context_description, bool allow_remove_constants) +{ + size_t columns = rhs.columns(); + if (lhs.columns() != columns) + return onError("Block structure mismatch in " + context_description + " stream: different number of columns:\n" + + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); + + for (size_t i = 0; i < columns; ++i) + { + const auto & actual = lhs.getByPosition(i); + const auto & expected = rhs.getByPosition(i); + + if constexpr (std::is_same_v) + { + if (!checkColumnStructure(actual, expected, context_description, allow_remove_constants, ErrorCodes::LOGICAL_ERROR)) + return false; + } + else + checkColumnStructure(actual, expected, context_description, allow_remove_constants, ErrorCodes::LOGICAL_ERROR); + } + + return ReturnType(true); } @@ -54,14 +133,22 @@ void Block::insert(size_t position, ColumnWithTypeAndName elem) if (name_pos.second >= position) ++name_pos.second; - index_by_name.emplace(elem.name, position); + auto [it, inserted] = index_by_name.emplace(elem.name, position); + if (!inserted) + checkColumnStructure(elem, data[it->second], + "(columns with identical name must have identical structure)", false, ErrorCodes::BAD_ARGUMENTS); + data.emplace(data.begin() + position, std::move(elem)); } void Block::insert(ColumnWithTypeAndName elem) { - index_by_name.emplace(elem.name, data.size()); + auto [it, inserted] = index_by_name.emplace(elem.name, data.size()); + if (!inserted) + checkColumnStructure(elem, data[it->second], + "(columns with identical name must have identical structure)", false, ErrorCodes::BAD_ARGUMENTS); + data.emplace_back(std::move(elem)); } @@ -473,67 +560,6 @@ DataTypes Block::getDataTypes() const } -template -static ReturnType checkBlockStructure(const Block & lhs, const Block & rhs, const std::string & context_description, bool allow_remove_constants) -{ - auto on_error = [](const std::string & message [[maybe_unused]], int code [[maybe_unused]]) - { - if constexpr (std::is_same_v) - throw Exception(message, code); - else - return false; - }; - - size_t columns = rhs.columns(); - if (lhs.columns() != columns) - return on_error("Block structure mismatch in " + context_description + " stream: different number of columns:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - - for (size_t i = 0; i < columns; ++i) - { - const auto & expected = rhs.getByPosition(i); - const auto & actual = lhs.getByPosition(i); - - if (actual.name != expected.name) - return on_error("Block structure mismatch in " + context_description + " stream: different names of columns:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - - if (!actual.type->equals(*expected.type)) - return on_error("Block structure mismatch in " + context_description + " stream: different types:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - - if (!actual.column || !expected.column) - continue; - - const IColumn * actual_column = actual.column.get(); - - /// If we allow to remove constants, and expected column is not const, then unwrap actual constant column. - if (allow_remove_constants && !isColumnConst(*expected.column)) - { - if (const auto * column_const = typeid_cast(actual_column)) - actual_column = &column_const->getDataColumn(); - } - - if (actual_column->getName() != expected.column->getName()) - return on_error("Block structure mismatch in " + context_description + " stream: different columns:\n" - + lhs.dumpStructure() + "\n" + rhs.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - - if (isColumnConst(*actual.column) && isColumnConst(*expected.column)) - { - Field actual_value = assert_cast(*actual.column).getField(); - Field expected_value = assert_cast(*expected.column).getField(); - - if (actual_value != expected_value) - return on_error("Block structure mismatch in " + context_description + " stream: different values of constants, actual: " - + applyVisitor(FieldVisitorToString(), actual_value) + ", expected: " + applyVisitor(FieldVisitorToString(), expected_value), - ErrorCodes::LOGICAL_ERROR); - } - } - - return ReturnType(true); -} - - bool blocksHaveEqualStructure(const Block & lhs, const Block & rhs) { return checkBlockStructure(lhs, rhs, {}, false); From b99cbd91810417a101186f60fb4675580d489d48 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 21 Jul 2021 04:45:57 +0300 Subject: [PATCH 013/220] Add a test --- tests/queries/0_stateless/01950_aliases_bad_cast.reference | 0 tests/queries/0_stateless/01950_aliases_bad_cast.sql | 2 ++ 2 files changed, 2 insertions(+) create mode 100644 tests/queries/0_stateless/01950_aliases_bad_cast.reference create mode 100644 tests/queries/0_stateless/01950_aliases_bad_cast.sql diff --git a/tests/queries/0_stateless/01950_aliases_bad_cast.reference b/tests/queries/0_stateless/01950_aliases_bad_cast.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/01950_aliases_bad_cast.sql b/tests/queries/0_stateless/01950_aliases_bad_cast.sql new file mode 100644 index 00000000000..a7265a1b020 --- /dev/null +++ b/tests/queries/0_stateless/01950_aliases_bad_cast.sql @@ -0,0 +1,2 @@ +SELECT 1, * FROM (SELECT NULL AS `1`); -- { serverError 36 } +SELECT '7', 'xyz', * FROM (SELECT NULL AS `'xyz'`); -- { serverError 36 } From 16eba6f0e7ac540f4a6ecb5562802c79422a31d1 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Jul 2021 02:22:01 +0300 Subject: [PATCH 014/220] Miscellaneous --- src/AggregateFunctions/UniqVariadicHash.h | 1 + src/Columns/ColumnTuple.h | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/AggregateFunctions/UniqVariadicHash.h b/src/AggregateFunctions/UniqVariadicHash.h index b3607a63285..94f54a7a059 100644 --- a/src/AggregateFunctions/UniqVariadicHash.h +++ b/src/AggregateFunctions/UniqVariadicHash.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB diff --git a/src/Columns/ColumnTuple.h b/src/Columns/ColumnTuple.h index 3f5422c7719..177ff6c412a 100644 --- a/src/Columns/ColumnTuple.h +++ b/src/Columns/ColumnTuple.h @@ -1,6 +1,6 @@ #pragma once -#include +#include namespace DB From 6e41a1b5caea52c2336338c4aeef1153b7aed5cf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Jul 2021 02:22:37 +0300 Subject: [PATCH 015/220] Fix error --- src/Functions/in.cpp | 4 ++-- src/Interpreters/Set.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 7cd9f64004d..17ec2518490 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -102,7 +102,7 @@ public: throw Exception("Second argument for function '" + getName() + "' must be Set; found " + column_set_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); - DB::Block columns_of_key_columns; + Block columns_of_key_columns; /// First argument may be a tuple or a single column. const ColumnWithTypeAndName & left_arg = arguments[0]; @@ -125,7 +125,7 @@ public: const DataTypes & tuple_types = type_tuple->getElements(); size_t tuple_size = tuple_columns.size(); for (size_t i = 0; i < tuple_size; ++i) - columns_of_key_columns.insert({ tuple_columns[i], tuple_types[i], "" }); + columns_of_key_columns.insert({ tuple_columns[i], tuple_types[i], "_" + toString(i) }); } else columns_of_key_columns.insert(left_arg); diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index ff502b499cd..8202c1ccce2 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -428,7 +428,7 @@ MergeTreeSetIndex::MergeTreeSetIndex(const Columns & set_elements, std::vector Date: Fri, 23 Jul 2021 02:37:38 +0300 Subject: [PATCH 016/220] Fix some tests --- .../0_stateless/01101_literal_column_clash.reference | 4 ---- tests/queries/0_stateless/01101_literal_column_clash.sql | 8 ++++---- tests/queries/0_stateless/01950_aliases_bad_cast.sql | 4 ++-- 3 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/01101_literal_column_clash.reference b/tests/queries/0_stateless/01101_literal_column_clash.reference index b89f59abb18..22844815f1e 100644 --- a/tests/queries/0_stateless/01101_literal_column_clash.reference +++ b/tests/queries/0_stateless/01101_literal_column_clash.reference @@ -3,9 +3,5 @@ 7 0 7 1 xyzabc 2 -1 3 1 2 0 0 -1 0 0 3 -\N 1 2 \N 0 -\N 1 0 \N 3 2 1 diff --git a/tests/queries/0_stateless/01101_literal_column_clash.sql b/tests/queries/0_stateless/01101_literal_column_clash.sql index 4a6064141ea..ea23f703f9f 100644 --- a/tests/queries/0_stateless/01101_literal_column_clash.sql +++ b/tests/queries/0_stateless/01101_literal_column_clash.sql @@ -7,13 +7,13 @@ join (select '1' as sid) as t2 on t2.sid = cast(t1.iid as String); select cast(7 as String), * from (select 3 "'String'"); select cast(7 as String), * from (select number "'String'" FROM numbers(2)); SELECT concat('xyz', 'abc'), * FROM (SELECT 2 AS "'xyz'"); -with 3 as "1" select 1, "1"; +with 3 as "1" select 1, "1"; -- { serverError 352 } -- https://github.com/ClickHouse/ClickHouse/issues/9953 select 1, * from (select 2 x) a left join (select 1, 3 y) b on y = x; -select 1, * from (select 2 x, 1) a right join (select 3 y) b on y = x; -select null, isConstant(null), * from (select 2 x) a left join (select null, 3 y) b on y = x; -select null, isConstant(null), * from (select 2 x, null) a right join (select 3 y) b on y = x; +select 1, * from (select 2 x, 1) a right join (select 3 y) b on y = x; -- { serverError 352 } +select null, isConstant(null), * from (select 2 x) a left join (select null, 3 y) b on y = x; -- { serverError 352 } +select null, isConstant(null), * from (select 2 x, null) a right join (select 3 y) b on y = x; -- { serverError 352 } -- other cases with joins and constants diff --git a/tests/queries/0_stateless/01950_aliases_bad_cast.sql b/tests/queries/0_stateless/01950_aliases_bad_cast.sql index a7265a1b020..bdd2339f855 100644 --- a/tests/queries/0_stateless/01950_aliases_bad_cast.sql +++ b/tests/queries/0_stateless/01950_aliases_bad_cast.sql @@ -1,2 +1,2 @@ -SELECT 1, * FROM (SELECT NULL AS `1`); -- { serverError 36 } -SELECT '7', 'xyz', * FROM (SELECT NULL AS `'xyz'`); -- { serverError 36 } +SELECT 1, * FROM (SELECT NULL AS `1`); -- { serverError 352 } +SELECT '7', 'xyz', * FROM (SELECT NULL AS `'xyz'`); -- { serverError 352 } From 5b69283a6ce110ae7222c6427e3a604ced8d91ea Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Jul 2021 03:25:26 +0300 Subject: [PATCH 017/220] Change error code --- src/Core/Block.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 6f106aa06f6..2aa06487df1 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -22,7 +22,7 @@ namespace ErrorCodes extern const int POSITION_OUT_OF_BOUND; extern const int NOT_FOUND_COLUMN_IN_BLOCK; extern const int SIZES_OF_COLUMNS_DOESNT_MATCH; - extern const int BAD_ARGUMENTS; + extern const int AMBIGUOUS_COLUMN_NAME; } template @@ -136,7 +136,7 @@ void Block::insert(size_t position, ColumnWithTypeAndName elem) auto [it, inserted] = index_by_name.emplace(elem.name, position); if (!inserted) checkColumnStructure(elem, data[it->second], - "(columns with identical name must have identical structure)", false, ErrorCodes::BAD_ARGUMENTS); + "(columns with identical name must have identical structure)", false, ErrorCodes::AMBIGUOUS_COLUMN_NAME); data.emplace(data.begin() + position, std::move(elem)); } @@ -147,7 +147,7 @@ void Block::insert(ColumnWithTypeAndName elem) auto [it, inserted] = index_by_name.emplace(elem.name, data.size()); if (!inserted) checkColumnStructure(elem, data[it->second], - "(columns with identical name must have identical structure)", false, ErrorCodes::BAD_ARGUMENTS); + "(columns with identical name must have identical structure)", false, ErrorCodes::AMBIGUOUS_COLUMN_NAME); data.emplace_back(std::move(elem)); } From 4bbbf58f3e74b7d89b64caeb246bd7afd3757076 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Fri, 23 Jul 2021 03:34:49 +0300 Subject: [PATCH 018/220] Fix test --- src/Interpreters/evaluateConstantExpression.cpp | 2 +- tests/queries/0_stateless/00818_alias_bug_4110.reference | 1 - tests/queries/0_stateless/00818_alias_bug_4110.sql | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index f814e1d8c02..97acb9aa6f6 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -213,7 +213,7 @@ namespace Disjunction result; - auto add_dnf = [&](const auto &dnf) + auto add_dnf = [&](const auto & dnf) { if (dnf.size() > limit) { diff --git a/tests/queries/0_stateless/00818_alias_bug_4110.reference b/tests/queries/0_stateless/00818_alias_bug_4110.reference index e6013d269c2..210fc67db66 100644 --- a/tests/queries/0_stateless/00818_alias_bug_4110.reference +++ b/tests/queries/0_stateless/00818_alias_bug_4110.reference @@ -4,7 +4,6 @@ 11 12 12 11 10 10 -10 11 11 12 11 10 12 11 12 diff --git a/tests/queries/0_stateless/00818_alias_bug_4110.sql b/tests/queries/0_stateless/00818_alias_bug_4110.sql index 7b2fd5d3864..df7e70cb275 100644 --- a/tests/queries/0_stateless/00818_alias_bug_4110.sql +++ b/tests/queries/0_stateless/00818_alias_bug_4110.sql @@ -5,7 +5,7 @@ select s.a + 1 as b, s.a + 2 as a from (select 10 as a) s; select s.a + 2 as b, s.a + 1 as a from (select 10 as a) s; select a, a as a from (select 10 as a); -select s.a, a, a + 1 as a from (select 10 as a) as s; +select s.a, a, a + 1 as a from (select 10 as a) as s; -- { serverError 352 } select s.a + 2 as b, b - 1 as a from (select 10 as a) s; select s.a as a, s.a + 2 as b from (select 10 as a) s; select s.a + 1 as a, s.a + 2 as b from (select 10 as a) s; From 760a998946ac70cea66472e23593d7f2d62ee5a1 Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Fri, 23 Jul 2021 15:38:02 +0300 Subject: [PATCH 019/220] DOCSUP-10607 --- docs/ru/sql-reference/functions/geo/h3.md | 36 +++++++++++++++ docs/ru/sql-reference/statements/system.md | 52 +++++++++++++++++++++- 2 files changed, 86 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 27a512a9931..d388850beea 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -193,6 +193,42 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; └────────────────────┘ ``` +## h3ToGeo {#h3togeo} + +Возвращает `(lon, lat)`, которые соответствуют уазанному индексу H3. + +**Синтаксис** + +``` sql +h3ToGeo(h3Index) +``` + +**Аргументы** + +- `h3Index` — H3 Index. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). + +**Возвращаемые значения** + +- `lon` — географическая долгота. Тип: [Float64](../../../sql-reference/data-types/float.md). +- `lat` — географическая широта. Тип: [Float64](../../../sql-reference/data-types/float.md). + + +**Пример** + +Запрос: + +``` sql +SELECT h3ToGeo(644325524701193974) coordinates; +``` + +Результат: + +``` text +┌─coordinates───────────────────────────┐ +│ (37.79506616830252,55.71290243145668) │ +└───────────────────────────────────────┘ +``` + ## h3kRing {#h3kring} Возвращает [H3](#h3index)-индексы шестигранников в радиусе `k` от данного в произвольном порядке. diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 634343d112f..d4c19b6ebf3 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -36,6 +36,7 @@ toc_title: SYSTEM - [START REPLICATION QUEUES](#query_language-system-start-replication-queues) - [SYNC REPLICA](#query_language-system-sync-replica) - [RESTART REPLICA](#query_language-system-restart-replica) +- [RESTORE REPLICA](#query_language-system-restore-replica) - [RESTART REPLICAS](#query_language-system-restart-replicas) ## RELOAD EMBEDDED DICTIONARIES] {#query_language-system-reload-emdedded-dictionaries} @@ -287,13 +288,60 @@ SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ### RESTART REPLICA {#query_language-system-restart-replica} -Реинициализация состояния Zookeeper-сессий для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с тем, что хранится в Zookeeper, как источник правды, и добавляет задачи в очередь репликации в Zookeeper, если необходимо. -Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при attach table. На короткое время таблица станет недоступной для любых операций. +Реинициализация состояния сессий Zookeeper для таблицы `ReplicatedMergeTree`. Сравнивает текущее состояние с тем, которое хранится в Zookeeper как исходным источником и при необходимости добавляет задачи в очередь репликации Zookeeper. +Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при `ATTACH TABLE`. Ненадолго таблица будет недоступна для любых операций. ``` sql SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name ``` +### RESTORE REPLICA {#query_language-system-restore-replica} + +Восстанавливает реплику, если данные (возможно) присутствуют, но метаданные Zookeeper потеряны. + +Работает только с таблицами `ReplicatedMergeTree` только для чтения. + +Запрос можно выполнить из: + + - корневого каталога ZooKeeper `/` с потерянными данными; + - каталога реплики `/replicas` с потерянными данными; + - конкретного пути в каталоге реплики `/replicas/replica_name/` с потерянными данными. + +К реплике прикрепляются локально найденные части, информация о них отправляется в Zookeeper. +Части, присутствующие в реплике до потери метаданных, не извлекаются повторно из других реплик, если они не устарели +(поэтому восстановление реплики не означает повторную загрузку всех данных по сети). + +Предупреждение: части в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. + +#### Синтаксис + +```sql +SYSTEM RESTORE REPLICA [db.]replicated_merge_tree_family_table_name [ON CLUSTER cluster_name] +``` + +Альтернативный синтаксис: + +```sql +SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_family_table_name +``` + +#### Пример + +```sql +-- Создание таблицы на нескольких серверах + +CREATE TABLE test(n UInt32) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}') +ORDER BY n PARTITION BY n % 10; + +INSERT INTO test SELECT * FROM numbers(1000); + +-- zookeeper_delete_path("/clickhouse/tables/test", recursive=True) <- root loss. + +SYSTEM RESTART REPLICA test; -- Table will attach as readonly as metadata is missing. +SYSTEM RESTORE REPLICA test; -- Need to execute on every replica, another way: RESTORE REPLICA test ON CLUSTER cluster +``` + ### RESTART REPLICAS {#query_language-system-restart-replicas} Реинициализация состояния ZooKeeper-сессий для всех `ReplicatedMergeTree` таблиц. Сравнивает текущее состояние реплики с тем, что хранится в ZooKeeper, как c источником правды, и добавляет задачи в очередь репликации в ZooKeeper, если необходимо. From 169e48c9780995efa1cae00490ad85438e4b3a6a Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Fri, 23 Jul 2021 15:53:53 +0300 Subject: [PATCH 020/220] DOCSUP-10607 --- docs/ru/sql-reference/functions/geo/h3.md | 2 +- docs/ru/sql-reference/statements/system.md | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index d388850beea..e7348a67270 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -195,7 +195,7 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; ## h3ToGeo {#h3togeo} -Возвращает `(lon, lat)`, которые соответствуют уазанному индексу H3. +Возвращает `(lon, lat)`, которые соответствуют указанному индексу H3. **Синтаксис** diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index d4c19b6ebf3..c9d81c0f60d 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -288,8 +288,8 @@ SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ### RESTART REPLICA {#query_language-system-restart-replica} -Реинициализация состояния сессий Zookeeper для таблицы `ReplicatedMergeTree`. Сравнивает текущее состояние с тем, которое хранится в Zookeeper как исходным источником и при необходимости добавляет задачи в очередь репликации Zookeeper. -Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при `ATTACH TABLE`. Ненадолго таблица будет недоступна для любых операций. +Реинициализирует состояние сессий Zookeeper для таблицы `ReplicatedMergeTree`. Сравнивает текущее состояние с тем, которое хранится в Zookeeper (как источнике правильных значений) и при необходимости добавляет задачи в очередь репликации Zookeeper. +Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при `ATTACH TABLE`. Какое-то время таблица будет недоступна для любых операций. ``` sql SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name @@ -299,7 +299,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name Восстанавливает реплику, если данные (возможно) присутствуют, но метаданные Zookeeper потеряны. -Работает только с таблицами `ReplicatedMergeTree` только для чтения. +Работает только с таблицами readonly `ReplicatedMergeTree`. Запрос можно выполнить из: @@ -308,10 +308,10 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name - конкретного пути в каталоге реплики `/replicas/replica_name/` с потерянными данными. К реплике прикрепляются локально найденные части, информация о них отправляется в Zookeeper. -Части, присутствующие в реплике до потери метаданных, не извлекаются повторно из других реплик, если они не устарели +Данные, присутствующие в реплике до потери метаданных, не извлекаются повторно из других реплик, если они не устарели (поэтому восстановление реплики не означает повторную загрузку всех данных по сети). -Предупреждение: части в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. +Предупреждение: потерянные данные в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. #### Синтаксис From 53aa74e3a82b8c84bf7141cd8f968f2051da87b3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Jul 2021 00:26:05 +0300 Subject: [PATCH 021/220] Another check just in case --- src/Core/Block.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 2aa06487df1..30774a12397 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -129,6 +129,9 @@ void Block::insert(size_t position, ColumnWithTypeAndName elem) throw Exception("Position out of bound in Block::insert(), max position = " + toString(data.size()), ErrorCodes::POSITION_OUT_OF_BOUND); + if (elem.name.empty()) + throw Exception("Column name in Block cannot be empty", ErrorCodes::AMBIGUOUS_COLUMN_NAME); + for (auto & name_pos : index_by_name) if (name_pos.second >= position) ++name_pos.second; @@ -144,6 +147,9 @@ void Block::insert(size_t position, ColumnWithTypeAndName elem) void Block::insert(ColumnWithTypeAndName elem) { + if (elem.name.empty()) + throw Exception("Column name in Block cannot be empty", ErrorCodes::AMBIGUOUS_COLUMN_NAME); + auto [it, inserted] = index_by_name.emplace(elem.name, data.size()); if (!inserted) checkColumnStructure(elem, data[it->second], @@ -155,6 +161,9 @@ void Block::insert(ColumnWithTypeAndName elem) void Block::insertUnique(ColumnWithTypeAndName elem) { + if (elem.name.empty()) + throw Exception("Column name in Block cannot be empty", ErrorCodes::AMBIGUOUS_COLUMN_NAME); + if (index_by_name.end() == index_by_name.find(elem.name)) insert(std::move(elem)); } From cbb686733c938ae3f497e70516c8788f9949b924 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Jul 2021 04:14:45 +0300 Subject: [PATCH 022/220] Fix ambiguous columns in test --- tests/queries/0_stateless/01236_graphite_mt.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01236_graphite_mt.sql b/tests/queries/0_stateless/01236_graphite_mt.sql index f3f1905b901..5981530bbf3 100644 --- a/tests/queries/0_stateless/01236_graphite_mt.sql +++ b/tests/queries/0_stateless/01236_graphite_mt.sql @@ -2,7 +2,7 @@ drop table if exists test_graphite; create table test_graphite (key UInt32, Path String, Time DateTime, Value Float64, Version UInt32, col UInt64) engine = GraphiteMergeTree('graphite_rollup') order by key settings index_granularity=10; insert into test_graphite -select 1, 'sum_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all +select 1 AS key, 'sum_1', toDateTime(today()) - number * 60 - 30, number, 1 AS Version, number from numbers(300) union all select 2, 'sum_1', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all select 1, 'sum_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all select 2, 'sum_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300) union all @@ -12,7 +12,7 @@ select 1, 'max_2', toDateTime(today()) - number * 60 - 30, number, 1, number fro select 2, 'max_2', toDateTime(today()) - number * 60 - 30, number, 1, number from numbers(300); insert into test_graphite -select 1, 'sum_1', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all +select 1 AS key, 'sum_1', toDateTime(today() - 3) - number * 60 - 30, number, 1 AS Version, number from numbers(1200) union all select 2, 'sum_1', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all select 1, 'sum_2', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all select 2, 'sum_2', toDateTime(today() - 3) - number * 60 - 30, number, 1, number from numbers(1200) union all From 04199ed81eaaef195084b54319cf67ac24a4c177 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Jul 2021 04:25:00 +0300 Subject: [PATCH 023/220] Fix the case of empty column name --- .../ExecuteScalarSubqueriesVisitor.cpp | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index f46cbdd2465..2b858512b98 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -80,9 +80,13 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr Block scalar; if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasScalar(scalar_query_hash_str)) + { scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str); + } else if (data.scalars.count(scalar_query_hash_str)) + { scalar = data.scalars[scalar_query_hash_str]; + } else { auto subquery_context = Context::createCopy(data.getContext()); @@ -149,7 +153,8 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); Block tmp_block; - while (tmp_block.rows() == 0 && executor.pull(tmp_block)); + while (tmp_block.rows() == 0 && executor.pull(tmp_block)) + ; if (tmp_block.rows() != 0) throw Exception("Scalar subquery returned more than one row", ErrorCodes::INCORRECT_RESULT_OF_SCALAR_SUBQUERY); @@ -173,10 +178,10 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr } else { - ColumnWithTypeAndName ctn; - ctn.type = std::make_shared(block.getDataTypes()); - ctn.column = ColumnTuple::create(block.getColumns()); - scalar.insert(ctn); + scalar.insert({ + ColumnTuple::create(block.getColumns()), + std::make_shared(block.getDataTypes()), + "tuple"}); } } From edfeb0957f24afd947eb02412c3d1b7fd869a95d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Jul 2021 04:52:18 +0300 Subject: [PATCH 024/220] Fix strange code --- .../evaluateConstantExpression.cpp | 26 ++++++++++++++++++- src/Storages/StorageDistributed.h | 5 ++-- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 97acb9aa6f6..a5fc29e32e2 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -121,6 +121,7 @@ std::tuple evaluateDatabaseNameForMergeEngine(const ASTPtr & node, return std::tuple{false, ast}; } + namespace { using Conjunction = ColumnsWithTypeAndName; @@ -368,7 +369,30 @@ std::optional evaluateExpressionOverConstantCondition(const ASTPtr & nod for (const auto & conjunct : dnf) { - Block block(conjunct); + Block block; + + for (const auto & elem : conjunct) + { + if (!block.has(elem.name)) + { + block.insert(elem); + } + else + { + /// Conjunction of condition on column equality to distinct values can never be satisfied. + + const ColumnWithTypeAndName & prev = block.getByName(elem.name); + + if (isColumnConst(*prev.column) && isColumnConst(*elem.column)) + { + Field prev_value = assert_cast(*prev.column).getField(); + Field curr_value = assert_cast(*elem.column).getField(); + + if (prev_value != curr_value) + return Blocks{}; + } + } + } // Block should contain all required columns from `target_expr` if (!has_required_columns(block)) diff --git a/src/Storages/StorageDistributed.h b/src/Storages/StorageDistributed.h index c63abbc6aa4..4660f7661cf 100644 --- a/src/Storages/StorageDistributed.h +++ b/src/Storages/StorageDistributed.h @@ -173,8 +173,9 @@ private: /// - optimize_skip_unused_shards /// - force_optimize_skip_unused_shards ClusterPtr getOptimizedCluster(ContextPtr, const StorageMetadataPtr & metadata_snapshot, const ASTPtr & query_ptr) const; - ClusterPtr - skipUnusedShards(ClusterPtr cluster, const ASTPtr & query_ptr, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) const; + + ClusterPtr skipUnusedShards( + ClusterPtr cluster, const ASTPtr & query_ptr, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) const; size_t getRandomShardIndex(const Cluster::ShardsInfo & shards); From a4b61819206d5bf230d14a9b59b725161428af3d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Jul 2021 05:07:37 +0300 Subject: [PATCH 025/220] Fix weirdness --- src/Storages/StorageMerge.cpp | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 43838b1d8c5..6b44d89d707 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -435,11 +435,17 @@ Pipe StorageMerge::createSources( if (!pipe.empty()) { if (concat_streams && pipe.numOutputPorts() > 1) + { // It's possible to have many tables read from merge, resize(1) might open too many files at the same time. // Using concat instead. pipe.addTransform(std::make_shared(pipe.getHeader(), pipe.numOutputPorts())); + } - if (has_database_virtual_column) + /// Add virtual columns if we don't already have them. + + Block pipe_header = pipe.getHeader(); + + if (has_database_virtual_column && !pipe_header.has("_database")) { ColumnWithTypeAndName column; column.name = "_database"; @@ -457,7 +463,7 @@ Pipe StorageMerge::createSources( }); } - if (has_table_virtual_column) + if (has_table_virtual_column && !pipe_header.has("_table")) { ColumnWithTypeAndName column; column.name = "_table"; From a0ed37e04e0a2b4550f0676254a7c348a43db670 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 28 Jul 2021 16:35:02 +0300 Subject: [PATCH 026/220] Ignore constness in ExpressionActionsChain::JoinStep Fix 01064_pm_all_join_const_and_nullable with bad cast check --- src/Interpreters/ExpressionActions.cpp | 9 ++++++--- src/Interpreters/ExpressionAnalyzer.cpp | 4 ++-- src/Interpreters/TableJoin.cpp | 24 +++--------------------- src/Interpreters/TableJoin.h | 1 - 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 6797947a101..d8c008c6065 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -793,12 +793,15 @@ ExpressionActionsChain::JoinStep::JoinStep( : Step({}) , analyzed_join(std::move(analyzed_join_)) , join(std::move(join_)) - , result_columns(std::move(required_columns_)) { - for (const auto & column : result_columns) + for (const auto & column : required_columns_) required_columns.emplace_back(column.name, column.type); - analyzed_join->addJoinedColumnsAndCorrectTypes(result_columns); + NamesAndTypesList result_names_and_types = required_columns; + analyzed_join->addJoinedColumnsAndCorrectTypes(result_names_and_types); + for (const auto & [name, type] : result_names_and_types) + /// `column` is `nullptr` because we don't care on constness here, it may be changed in join + result_columns.emplace_back(nullptr, type, name); } void ExpressionActionsChain::JoinStep::finalize(const NameSet & required_output_) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 875a7bef862..d48cee413ae 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -215,7 +215,7 @@ void ExpressionAnalyzer::analyzeAggregation() if (join) { getRootActionsNoMakeSet(analyzedJoin().leftKeysList(), true, temp_actions, false); - auto sample_columns = temp_actions->getResultColumns(); + auto sample_columns = temp_actions->getNamesAndTypesList(); analyzedJoin().addJoinedColumnsAndCorrectTypes(sample_columns); temp_actions = std::make_shared(sample_columns); } @@ -1213,7 +1213,7 @@ void SelectQueryExpressionAnalyzer::appendSelect(ExpressionActionsChain & chain, } ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChain & chain, bool only_types, bool optimize_read_in_order, - ManyExpressionActions & order_by_elements_actions) + ManyExpressionActions & order_by_elements_actions) { const auto * select_query = getSelectQuery(); diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 20e8f6b18b4..86c84d9c8c9 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -231,20 +231,7 @@ void TableJoin::addJoinedColumn(const NameAndTypePair & joined_column) void TableJoin::addJoinedColumnsAndCorrectTypes(NamesAndTypesList & names_and_types, bool correct_nullability) const { - ColumnsWithTypeAndName columns; - for (auto & pair : names_and_types) - columns.emplace_back(nullptr, std::move(pair.type), std::move(pair.name)); - names_and_types.clear(); - - addJoinedColumnsAndCorrectTypes(columns, correct_nullability); - - for (auto & col : columns) - names_and_types.emplace_back(std::move(col.name), std::move(col.type)); -} - -void TableJoin::addJoinedColumnsAndCorrectTypes(ColumnsWithTypeAndName & columns, bool correct_nullability) const -{ - for (auto & col : columns) + for (auto & col : names_and_types) { if (hasUsing()) { @@ -252,17 +239,12 @@ void TableJoin::addJoinedColumnsAndCorrectTypes(ColumnsWithTypeAndName & columns col.type = it->second; } if (correct_nullability && leftBecomeNullable(col.type)) - { - /// No need to nullify constants - bool is_column_const = col.column && isColumnConst(*col.column); - if (!is_column_const) - col.type = JoinCommon::convertTypeToNullable(col.type); - } + col.type = JoinCommon::convertTypeToNullable(col.type); } /// Types in columns_added_by_join already converted and set nullable if needed for (const auto & col : columns_added_by_join) - columns.emplace_back(nullptr, col.type, col.name); + names_and_types.emplace_back(col.name, col.type); } bool TableJoin::sameStrictnessAndKind(ASTTableJoin::Strictness strictness_, ASTTableJoin::Kind kind_) const diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 4c8c16028f5..4fe9565666f 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -191,7 +191,6 @@ public: void addJoinedColumn(const NameAndTypePair & joined_column); void addJoinedColumnsAndCorrectTypes(NamesAndTypesList & names_and_types, bool correct_nullability = true) const; - void addJoinedColumnsAndCorrectTypes(ColumnsWithTypeAndName & columns, bool correct_nullability = true) const; /// Calculates common supertypes for corresponding join key columns. bool inferJoinKeyCommonType(const NamesAndTypesList & left, const NamesAndTypesList & right); From 9af47eeb987aa8e57ae256ea66916d09354bc494 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Wed, 28 Jul 2021 22:07:41 +0300 Subject: [PATCH 027/220] Update h3.md --- docs/en/sql-reference/functions/geo/h3.md | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 6c03f55cebe..fe36238bbef 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -197,7 +197,7 @@ Result: ## h3ToGeo {#h3togeo} -Returns `(lon, lat)` that corresponds to the provided H3 index. +Returns `(lon, lat)` that corresponds to the provided [H3](#h3index) index. **Syntax** @@ -207,20 +207,18 @@ h3ToGeo(h3Index) **Arguments** -- `h3Index` — H3 Index. Type: [UInt64](../../../sql-reference/data-types/int-uint.md). +- `h3Index` — H3 Index. [UInt64](../../../sql-reference/data-types/int-uint.md). **Returned values** -- `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md). -- `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md). - +- A tuple consisting of two values: `tuple(lon,lat)`. `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md). `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md). **Example** Query: ``` sql -SELECT h3ToGeo(644325524701193974) coordinates; +SELECT h3ToGeo(644325524701193974) AS coordinates; ``` Result: @@ -230,6 +228,7 @@ Result: │ (37.79506616830252,55.71290243145668) │ └───────────────────────────────────────┘ ``` + ## h3kRing {#h3kring} Lists all the [H3](#h3index) hexagons in the raduis of `k` from the given hexagon in random order. From 72e09644d31ea8a286c76ead2e5227ab996a1aaa Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Wed, 28 Jul 2021 22:13:29 +0300 Subject: [PATCH 028/220] Update h3.md --- docs/ru/sql-reference/functions/geo/h3.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index e7348a67270..e04528f39fe 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -195,7 +195,7 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; ## h3ToGeo {#h3togeo} -Возвращает `(lon, lat)`, которые соответствуют указанному индексу H3. +Возвращает координаты широты и долготы, которые соответствуют указанному индексу H3. **Синтаксис** @@ -209,16 +209,16 @@ h3ToGeo(h3Index) **Возвращаемые значения** +- Аналогично EN версии? - `lon` — географическая долгота. Тип: [Float64](../../../sql-reference/data-types/float.md). - `lat` — географическая широта. Тип: [Float64](../../../sql-reference/data-types/float.md). - **Пример** Запрос: ``` sql -SELECT h3ToGeo(644325524701193974) coordinates; +SELECT h3ToGeo(644325524701193974) AS coordinates; ``` Результат: From 473641225df347973810bac889c28fb8eb9479b1 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Wed, 28 Jul 2021 22:19:51 +0300 Subject: [PATCH 029/220] Update h3.md --- docs/en/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index fe36238bbef..df9df7f3bd0 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -197,7 +197,7 @@ Result: ## h3ToGeo {#h3togeo} -Returns `(lon, lat)` that corresponds to the provided [H3](#h3index) index. +Returns the geographical coordinates of latitude and longitude corresponding to the provided [H3](#h3index) index. **Syntax** From 976be3be6a3520d2cc68dbbfdd9ecc9ecbb68628 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Wed, 28 Jul 2021 22:24:04 +0300 Subject: [PATCH 030/220] Update h3.md --- docs/ru/sql-reference/functions/geo/h3.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index e04528f39fe..801fe947385 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -195,7 +195,7 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; ## h3ToGeo {#h3togeo} -Возвращает координаты широты и долготы, которые соответствуют указанному индексу H3. +Возвращает координаты широты и долготы, которые соответствуют указанному [H3](#h3index)-индексу. **Синтаксис** @@ -205,7 +205,7 @@ h3ToGeo(h3Index) **Аргументы** -- `h3Index` — H3 Index. Тип: [UInt64](../../../sql-reference/data-types/int-uint.md). +- `h3Index` — [H3](#h3index)-индекс. [UInt64](../../../sql-reference/data-types/int-uint.md). **Возвращаемые значения** From db310e3b6395b5e8d830f2840e2bffb005096ddc Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Wed, 28 Jul 2021 22:25:31 +0300 Subject: [PATCH 031/220] Update h3.md --- docs/ru/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 801fe947385..b23bb99ce67 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -195,7 +195,7 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; ## h3ToGeo {#h3togeo} -Возвращает координаты широты и долготы, которые соответствуют указанному [H3](#h3index)-индексу. +Возвращает координаты широты и долготы, соответствующие указанному [H3](#h3index)-индексу. **Синтаксис** From e0b345d99acd03d67e87222fce8bfe7a19ababa5 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Thu, 29 Jul 2021 10:44:41 +0300 Subject: [PATCH 032/220] Update system.md --- docs/ru/sql-reference/statements/system.md | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index c9d81c0f60d..b1d5b5e0f04 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -288,8 +288,7 @@ SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ### RESTART REPLICA {#query_language-system-restart-replica} -Реинициализирует состояние сессий Zookeeper для таблицы `ReplicatedMergeTree`. Сравнивает текущее состояние с тем, которое хранится в Zookeeper (как источнике правильных значений) и при необходимости добавляет задачи в очередь репликации Zookeeper. -Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при `ATTACH TABLE`. Какое-то время таблица будет недоступна для любых операций. +Реинициализирует состояние сессий Zookeeper для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с Zookeeper (как источнике правильных значений) и при необходимости добавляет задачи в очередь репликации Zookeeper. В процессе инициализации очереди репликации на основе данных ZooKeeper, какое-то время таблица будет недоступна для любых операций. ``` sql SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name @@ -297,9 +296,9 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name ### RESTORE REPLICA {#query_language-system-restore-replica} -Восстанавливает реплику, если данные (возможно) присутствуют, но метаданные Zookeeper потеряны. +Восстанавливает реплику, если метаданные Zookeeper потеряны, но сами данные возможно существуют. -Работает только с таблицами readonly `ReplicatedMergeTree`. +Работает только с таблицами семейства `ReplicatedMergeTree` в режиме только на чтение. Запрос можно выполнить из: @@ -308,8 +307,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name - конкретного пути в каталоге реплики `/replicas/replica_name/` с потерянными данными. К реплике прикрепляются локально найденные части, информация о них отправляется в Zookeeper. -Данные, присутствующие в реплике до потери метаданных, не извлекаются повторно из других реплик, если они не устарели -(поэтому восстановление реплики не означает повторную загрузку всех данных по сети). +Данные, присутствующие в реплике до потери метаданных, не извлекаются повторно из других реплик, если они не устарели (поэтому восстановление реплики не означает повторную загрузку всех данных по сети). Предупреждение: потерянные данные в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. @@ -338,8 +336,14 @@ INSERT INTO test SELECT * FROM numbers(1000); -- zookeeper_delete_path("/clickhouse/tables/test", recursive=True) <- root loss. -SYSTEM RESTART REPLICA test; -- Table will attach as readonly as metadata is missing. -SYSTEM RESTORE REPLICA test; -- Need to execute on every replica, another way: RESTORE REPLICA test ON CLUSTER cluster +SYSTEM RESTART REPLICA test; -- таблица будет прикреплена только для чтения, так как метаданные отсутствуют. +SYSTEM RESTORE REPLICA test; -- необходимо выполнить на каждой реплике. +``` + +Альтернативный способ: + +```sql +RESTORE REPLICA test ON CLUSTER cluster; ``` ### RESTART REPLICAS {#query_language-system-restart-replicas} From 0a3d021f843c513328fd478923191b2185a05cad Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Thu, 29 Jul 2021 10:44:44 +0300 Subject: [PATCH 033/220] Update system.md --- docs/en/sql-reference/statements/system.md | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index d1526c10203..57f92296ffa 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -311,8 +311,7 @@ One may execute query after: - Individual replica path `/replicas/replica_name/` loss. Replica attaches locally found parts and sends info about them to Zookeeper. -Parts present on replica before metadata loss are not re-fetched from other replicas if not being outdated -(so replica restoration does not mean re-downloading all data over the network). +Parts present on a replica before metadata loss are not re-fetched from other ones if not being outdated (so replica restoration does not mean re-downloading all data over the network). Caveat: parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached. @@ -342,7 +341,12 @@ INSERT INTO test SELECT * FROM numbers(1000); -- zookeeper_delete_path("/clickhouse/tables/test", recursive=True) <- root loss. SYSTEM RESTART REPLICA test; -- Table will attach as readonly as metadata is missing. -SYSTEM RESTORE REPLICA test; -- Need to execute on every replica, another way: RESTORE REPLICA test ON CLUSTER cluster +SYSTEM RESTORE REPLICA test; -- Need to execute on every replica. +``` + +Another way: +```sql +RESTORE REPLICA test ON CLUSTER cluster; ``` ### RESTART REPLICAS {#query_language-system-restart-replicas} From f0239672248f7601a214e00a425c998c6c7777a5 Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Thu, 29 Jul 2021 12:04:50 +0300 Subject: [PATCH 034/220] Update h3.md --- docs/en/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index df9df7f3bd0..7b092aba24d 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -197,7 +197,7 @@ Result: ## h3ToGeo {#h3togeo} -Returns the geographical coordinates of latitude and longitude corresponding to the provided [H3](#h3index) index. +Returns the geographical coordinates of longitude and latitude corresponding to the provided [H3](#h3index) index. **Syntax** From bf4b8d3d5ba62be81f48b3c70b87655a0469adce Mon Sep 17 00:00:00 2001 From: Roman Bug Date: Thu, 29 Jul 2021 12:08:11 +0300 Subject: [PATCH 035/220] Update h3.md --- docs/ru/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index b23bb99ce67..725190359e4 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -195,7 +195,7 @@ SELECT geoToH3(37.79506683, 55.71290588, 15) as h3Index; ## h3ToGeo {#h3togeo} -Возвращает координаты широты и долготы, соответствующие указанному [H3](#h3index)-индексу. +Возвращает географические координаты долготы и широты, соответствующие указанному [H3](#h3index)-индексу. **Синтаксис** From 3d3b1658c559909dd70dc6143a043739f99e5adc Mon Sep 17 00:00:00 2001 From: zhangxiao871 Date: Tue, 3 Aug 2021 17:59:08 +0800 Subject: [PATCH 036/220] Fix clickhouse-keeper create znode exists and empty condition. --- src/Coordination/KeeperStorage.cpp | 210 +++++++++--------- .../test_keeper_back_to_back/test.py | 39 ++++ 2 files changed, 144 insertions(+), 105 deletions(-) diff --git a/src/Coordination/KeeperStorage.cpp b/src/Coordination/KeeperStorage.cpp index 4c3f649a6b6..320754c7d31 100644 --- a/src/Coordination/KeeperStorage.cpp +++ b/src/Coordination/KeeperStorage.cpp @@ -248,117 +248,117 @@ struct KeeperStorageCreateRequest final : public KeeperStorageRequest Coordination::ZooKeeperCreateResponse & response = dynamic_cast(*response_ptr); Coordination::ZooKeeperCreateRequest & request = dynamic_cast(*zk_request); - if (container.contains(request.path)) + auto parent_path = parentPath(request.path); + auto it = container.find(parent_path); + + if (it == container.end()) + { + response.error = Coordination::Error::ZNONODE; + return { response_ptr, undo }; + } + else if (it->value.stat.ephemeralOwner != 0) + { + response.error = Coordination::Error::ZNOCHILDRENFOREPHEMERALS; + return { response_ptr, undo }; + } + std::string path_created = request.path; + if (request.is_sequential) + { + auto seq_num = it->value.seq_num; + + std::stringstream seq_num_str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM + seq_num_str.exceptions(std::ios::failbit); + seq_num_str << std::setw(10) << std::setfill('0') << seq_num; + + path_created += seq_num_str.str(); + } + if (container.contains(path_created)) { response.error = Coordination::Error::ZNODEEXISTS; + return { response_ptr, undo }; } - else + auto child_path = getBaseName(path_created); + if (child_path.empty()) { - auto parent_path = parentPath(request.path); - auto it = container.find(parent_path); - - if (it == container.end()) - { - response.error = Coordination::Error::ZNONODE; - } - else if (it->value.stat.ephemeralOwner != 0) - { - response.error = Coordination::Error::ZNOCHILDRENFOREPHEMERALS; - } - else - { - auto & session_auth_ids = storage.session_and_auth[session_id]; - - KeeperStorage::Node created_node; - - Coordination::ACLs node_acls; - if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log)) - { - response.error = Coordination::Error::ZINVALIDACL; - return {response_ptr, {}}; - } - - uint64_t acl_id = storage.acl_map.convertACLs(node_acls); - storage.acl_map.addUsage(acl_id); - - created_node.acl_id = acl_id; - created_node.stat.czxid = zxid; - created_node.stat.mzxid = zxid; - created_node.stat.pzxid = zxid; - created_node.stat.ctime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); - created_node.stat.mtime = created_node.stat.ctime; - created_node.stat.numChildren = 0; - created_node.stat.dataLength = request.data.length(); - created_node.stat.ephemeralOwner = request.is_ephemeral ? session_id : 0; - created_node.data = request.data; - created_node.is_sequental = request.is_sequential; - - std::string path_created = request.path; - - if (request.is_sequential) - { - auto seq_num = it->value.seq_num; - - std::stringstream seq_num_str; // STYLE_CHECK_ALLOW_STD_STRING_STREAM - seq_num_str.exceptions(std::ios::failbit); - seq_num_str << std::setw(10) << std::setfill('0') << seq_num; - - path_created += seq_num_str.str(); - } - - int32_t parent_cversion = request.parent_cversion; - auto child_path = getBaseName(path_created); - int64_t prev_parent_zxid; - int32_t prev_parent_cversion; - container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid, - parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent) - { - - parent.children.insert(child_path); - prev_parent_cversion = parent.stat.cversion; - prev_parent_zxid = parent.stat.pzxid; - - /// Increment sequential number even if node is not sequential - ++parent.seq_num; - - if (parent_cversion == -1) - ++parent.stat.cversion; - else if (parent_cversion > parent.stat.cversion) - parent.stat.cversion = parent_cversion; - - if (zxid > parent.stat.pzxid) - parent.stat.pzxid = zxid; - ++parent.stat.numChildren; - }); - - response.path_created = path_created; - container.insert(path_created, std::move(created_node)); - - if (request.is_ephemeral) - ephemerals[session_id].emplace(path_created); - - undo = [&storage, prev_parent_zxid, prev_parent_cversion, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id] - { - storage.container.erase(path_created); - storage.acl_map.removeUsage(acl_id); - - if (is_ephemeral) - storage.ephemerals[session_id].erase(path_created); - - storage.container.updateValue(parent_path, [child_path, prev_parent_zxid, prev_parent_cversion] (KeeperStorage::Node & undo_parent) - { - --undo_parent.stat.numChildren; - --undo_parent.seq_num; - undo_parent.stat.cversion = prev_parent_cversion; - undo_parent.stat.pzxid = prev_parent_zxid; - undo_parent.children.erase(child_path); - }); - }; - - response.error = Coordination::Error::ZOK; - } + response.error = Coordination::Error::ZBADARGUMENTS; + return { response_ptr, undo }; } + auto & session_auth_ids = storage.session_and_auth[session_id]; + + KeeperStorage::Node created_node; + + Coordination::ACLs node_acls; + if (!fixupACL(request.acls, session_auth_ids, node_acls, !request.restored_from_zookeeper_log)) + { + response.error = Coordination::Error::ZINVALIDACL; + return {response_ptr, {}}; + } + + uint64_t acl_id = storage.acl_map.convertACLs(node_acls); + storage.acl_map.addUsage(acl_id); + + created_node.acl_id = acl_id; + created_node.stat.czxid = zxid; + created_node.stat.mzxid = zxid; + created_node.stat.pzxid = zxid; + created_node.stat.ctime = std::chrono::system_clock::now().time_since_epoch() / std::chrono::milliseconds(1); + created_node.stat.mtime = created_node.stat.ctime; + created_node.stat.numChildren = 0; + created_node.stat.dataLength = request.data.length(); + created_node.stat.ephemeralOwner = request.is_ephemeral ? session_id : 0; + created_node.data = request.data; + created_node.is_sequental = request.is_sequential; + + int32_t parent_cversion = request.parent_cversion; + int64_t prev_parent_zxid; + int32_t prev_parent_cversion; + container.updateValue(parent_path, [child_path, zxid, &prev_parent_zxid, + parent_cversion, &prev_parent_cversion] (KeeperStorage::Node & parent) + { + + parent.children.insert(child_path); + prev_parent_cversion = parent.stat.cversion; + prev_parent_zxid = parent.stat.pzxid; + + /// Increment sequential number even if node is not sequential + ++parent.seq_num; + + if (parent_cversion == -1) + ++parent.stat.cversion; + else if (parent_cversion > parent.stat.cversion) + parent.stat.cversion = parent_cversion; + + if (zxid > parent.stat.pzxid) + parent.stat.pzxid = zxid; + ++parent.stat.numChildren; + }); + + response.path_created = path_created; + container.insert(path_created, std::move(created_node)); + + if (request.is_ephemeral) + ephemerals[session_id].emplace(path_created); + + undo = [&storage, prev_parent_zxid, prev_parent_cversion, session_id, path_created, is_ephemeral = request.is_ephemeral, parent_path, child_path, acl_id] + { + storage.container.erase(path_created); + storage.acl_map.removeUsage(acl_id); + + if (is_ephemeral) + storage.ephemerals[session_id].erase(path_created); + + storage.container.updateValue(parent_path, [child_path, prev_parent_zxid, prev_parent_cversion] (KeeperStorage::Node & undo_parent) + { + --undo_parent.stat.numChildren; + --undo_parent.seq_num; + undo_parent.stat.cversion = prev_parent_cversion; + undo_parent.stat.pzxid = prev_parent_zxid; + undo_parent.children.erase(child_path); + }); + }; + + response.error = Coordination::Error::ZOK; return { response_ptr, undo }; } }; diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index 41c270e05e8..48af4de4198 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -90,6 +90,45 @@ def test_sequential_nodes(started_cluster): genuine_childs = list(sorted(genuine_zk.get_children("/test_sequential_nodes"))) fake_childs = list(sorted(fake_zk.get_children("/test_sequential_nodes"))) assert genuine_childs == fake_childs + + genuine_zk.create("/test_sequential_nodes_1") + fake_zk.create("/test_sequential_nodes_1") + + genuine_zk.create("/test_sequential_nodes_1/a", sequence=True) + fake_zk.create("/test_sequential_nodes_1/a", sequence=True) + + genuine_zk.create("/test_sequential_nodes_1/a0000000002") + fake_zk.create("/test_sequential_nodes_1/a0000000002") + + genuine_throw = False + fake_throw = False + try: + genuine_zk.create("/test_sequential_nodes_1/a", sequence=True) + except Exception as ex: + genuine_throw = True + + try: + fake_zk.create("/test_sequential_nodes_1/a", sequence=True) + except Exception as ex: + fake_throw = True + + assert genuine_throw == fake_throw + + genuine_childs_1 = list(sorted(genuine_zk.get_children("/test_sequential_nodes_1"))) + fake_childs_1 = list(sorted(fake_zk.get_children("/test_sequential_nodes_1"))) + assert genuine_childs_1 == fake_childs_1 + + genuine_zk.create("/test_sequential_nodes_2") + fake_zk.create("/test_sequential_nodes_2") + + genuine_zk.create("/test_sequential_nodes_2/node") + fake_zk.create("/test_sequential_nodes_2/node") + genuine_zk.create("/test_sequential_nodes_2/node", sequence=True) + fake_zk.create("/test_sequential_nodes_2/node", sequence=True) + + genuine_childs_2 = list(sorted(genuine_zk.get_children("/test_sequential_nodes_2"))) + fake_childs_2 = list(sorted(fake_zk.get_children("/test_sequential_nodes_2"))) + assert genuine_childs_2 == fake_childs_2 finally: for zk in [genuine_zk, fake_zk]: stop_zk(zk) From 5a33c81c9b1b9a8974dd8ce05b416b0a15098f9b Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Wed, 4 Aug 2021 15:34:54 +0300 Subject: [PATCH 037/220] Update system.md --- docs/ru/sql-reference/statements/system.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index b1d5b5e0f04..eae64c047a4 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -298,7 +298,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name Восстанавливает реплику, если метаданные Zookeeper потеряны, но сами данные возможно существуют. -Работает только с таблицами семейства `ReplicatedMergeTree` в режиме только на чтение. +Работает только с таблицами семейства `ReplicatedMergeTree` и только в режиме чтения. Запрос можно выполнить из: @@ -307,7 +307,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name - конкретного пути в каталоге реплики `/replicas/replica_name/` с потерянными данными. К реплике прикрепляются локально найденные части, информация о них отправляется в Zookeeper. -Данные, присутствующие в реплике до потери метаданных, не извлекаются повторно из других реплик, если они не устарели (поэтому восстановление реплики не означает повторную загрузку всех данных по сети). +Если присутствующие в реплике до потери метаданных данные не устарели, они не извлекаются повторно из других реплик. Поэтому восстановление реплики не означает повторную загрузку всех данных по сети. Предупреждение: потерянные данные в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. From 01ecf25b39d3e476d8f39dafb821c751509ec532 Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Wed, 4 Aug 2021 16:09:59 +0300 Subject: [PATCH 038/220] Update h3.md --- docs/ru/sql-reference/functions/geo/h3.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 725190359e4..5f50e84fc0c 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -218,7 +218,7 @@ h3ToGeo(h3Index) Запрос: ``` sql -SELECT h3ToGeo(644325524701193974) AS coordinates; +SELECT h3ToGeo(644325524701193974) coordinates; ``` Результат: From 72e868388aba1132d5839e8f782723aa364b45de Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Wed, 4 Aug 2021 17:29:54 +0300 Subject: [PATCH 039/220] Update system.md --- docs/ru/sql-reference/statements/system.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index eae64c047a4..c2cac5d1457 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -288,7 +288,7 @@ SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ### RESTART REPLICA {#query_language-system-restart-replica} -Реинициализирует состояние сессий Zookeeper для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с Zookeeper (как источнике правильных значений) и при необходимости добавляет задачи в очередь репликации Zookeeper. В процессе инициализации очереди репликации на основе данных ZooKeeper, какое-то время таблица будет недоступна для любых операций. +Реинициализирует состояние сессий Zookeeper для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с Zookeeper (как с эталоном) и при необходимости добавляет задачи в очередь репликации Zookeeper. В процессе инициализации очереди репликации на основе данных ZooKeeper, какое-то время таблица будет недоступна для любых операций. ``` sql SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name @@ -311,7 +311,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name Предупреждение: потерянные данные в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. -#### Синтаксис +**Синтаксис** ```sql SYSTEM RESTORE REPLICA [db.]replicated_merge_tree_family_table_name [ON CLUSTER cluster_name] @@ -323,7 +323,7 @@ SYSTEM RESTORE REPLICA [db.]replicated_merge_tree_family_table_name [ON CLUSTER SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_family_table_name ``` -#### Пример +**Пример** ```sql -- Создание таблицы на нескольких серверах From 3a27b724d09c7d43cd738a7c47a35b8f810c4fde Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Wed, 4 Aug 2021 22:47:38 +0300 Subject: [PATCH 040/220] edit h3ToGeo function --- docs/en/sql-reference/functions/geo/h3.md | 2 +- docs/en/sql-reference/statements/system.md | 3 ++- docs/ru/sql-reference/functions/geo/h3.md | 4 +--- docs/ru/sql-reference/statements/system.md | 3 ++- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/en/sql-reference/functions/geo/h3.md b/docs/en/sql-reference/functions/geo/h3.md index 7b092aba24d..2d31ef0710e 100644 --- a/docs/en/sql-reference/functions/geo/h3.md +++ b/docs/en/sql-reference/functions/geo/h3.md @@ -211,7 +211,7 @@ h3ToGeo(h3Index) **Returned values** -- A tuple consisting of two values: `tuple(lon,lat)`. `lon` — Longitude. Type: [Float64](../../../sql-reference/data-types/float.md). `lat` — Latitude. Type: [Float64](../../../sql-reference/data-types/float.md). +- A tuple consisting of two values: `tuple(lon,lat)`. `lon` — Longitude. [Float64](../../../sql-reference/data-types/float.md). `lat` — Latitude. [Float64](../../../sql-reference/data-types/float.md). **Example** diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 57f92296ffa..b9ec779beb9 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -313,7 +313,8 @@ One may execute query after: Replica attaches locally found parts and sends info about them to Zookeeper. Parts present on a replica before metadata loss are not re-fetched from other ones if not being outdated (so replica restoration does not mean re-downloading all data over the network). -Caveat: parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached. +!!! warning "Caveat" +Parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached. #### Syntax diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 5f50e84fc0c..2d33c6ba15a 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -209,9 +209,7 @@ h3ToGeo(h3Index) **Возвращаемые значения** -- Аналогично EN версии? -- `lon` — географическая долгота. Тип: [Float64](../../../sql-reference/data-types/float.md). -- `lat` — географическая широта. Тип: [Float64](../../../sql-reference/data-types/float.md). +- Набор из двух значений: `tuple(lon,lat)`. `lon` — долгота. [Float64](../../../sql-reference/data-types/float.md). `lat` — широта. [Float64](../../../sql-reference/data-types/float.md). **Пример** diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index c2cac5d1457..7b69d3897ca 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -309,7 +309,8 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name К реплике прикрепляются локально найденные части, информация о них отправляется в Zookeeper. Если присутствующие в реплике до потери метаданных данные не устарели, они не извлекаются повторно из других реплик. Поэтому восстановление реплики не означает повторную загрузку всех данных по сети. -Предупреждение: потерянные данные в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. +!!! warning "Предупреждение" + Потерянные данные в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. **Синтаксис** From e76e9abb2d8d57f72b0a7705d369ccb585c85b52 Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Wed, 4 Aug 2021 23:07:38 +0300 Subject: [PATCH 041/220] edit RESTORE REPLICA query --- docs/en/sql-reference/statements/system.md | 13 +++++++------ docs/ru/sql-reference/statements/system.md | 11 ++++++----- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index b9ec779beb9..3d5a4fe4905 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -316,7 +316,7 @@ Parts present on a replica before metadata loss are not re-fetched from other on !!! warning "Caveat" Parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached. -#### Syntax +**Syntax** ```sql SYSTEM RESTORE REPLICA [db.]replicated_merge_tree_family_table_name [ON CLUSTER cluster_name] @@ -328,11 +328,11 @@ Alternative syntax: SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_family_table_name ``` -#### Example +**Example** + +Creating table on multiple servers. After the replica's root directory is lost, the table will will attach as readonly as metadata is missing. The last query need to execute on every replica. ```sql --- Creating table on multiple servers - CREATE TABLE test(n UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}') ORDER BY n PARTITION BY n % 10; @@ -341,11 +341,12 @@ INSERT INTO test SELECT * FROM numbers(1000); -- zookeeper_delete_path("/clickhouse/tables/test", recursive=True) <- root loss. -SYSTEM RESTART REPLICA test; -- Table will attach as readonly as metadata is missing. -SYSTEM RESTORE REPLICA test; -- Need to execute on every replica. +SYSTEM RESTART REPLICA test; +SYSTEM RESTORE REPLICA test; ``` Another way: + ```sql RESTORE REPLICA test ON CLUSTER cluster; ``` diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 7b69d3897ca..14ff974ee33 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -288,7 +288,8 @@ SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ### RESTART REPLICA {#query_language-system-restart-replica} -Реинициализирует состояние сессий Zookeeper для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с Zookeeper (как с эталоном) и при необходимости добавляет задачи в очередь репликации Zookeeper. В процессе инициализации очереди репликации на основе данных ZooKeeper, какое-то время таблица будет недоступна для любых операций. +Реинициализирует состояние сессий Zookeeper для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с Zookeeper (как с эталоном) и при необходимости добавляет задачи в очередь репликации Zookeeper. +Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при `ATTACH TABLE`. Некоторое время таблица будет недоступна для любых операций. ``` sql SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name @@ -326,9 +327,9 @@ SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_fami **Пример** -```sql --- Создание таблицы на нескольких серверах +Создание таблицы на нескольких серверах. После потери корневого каталога реплики таблица будет прикреплена только для чтения, так как метаданные отсутствуют. Последний запрос необходимо выполнить на каждой реплике. +```sql CREATE TABLE test(n UInt32) ENGINE = ReplicatedMergeTree('/clickhouse/tables/test/', '{replica}') ORDER BY n PARTITION BY n % 10; @@ -337,8 +338,8 @@ INSERT INTO test SELECT * FROM numbers(1000); -- zookeeper_delete_path("/clickhouse/tables/test", recursive=True) <- root loss. -SYSTEM RESTART REPLICA test; -- таблица будет прикреплена только для чтения, так как метаданные отсутствуют. -SYSTEM RESTORE REPLICA test; -- необходимо выполнить на каждой реплике. +SYSTEM RESTART REPLICA test; +SYSTEM RESTORE REPLICA test; ``` Альтернативный способ: From 5b08a73d5297a1899712142553285d650e4675ca Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Wed, 4 Aug 2021 23:28:25 +0300 Subject: [PATCH 042/220] edit warning in system.md --- docs/en/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 3d5a4fe4905..153db6963a0 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -314,7 +314,7 @@ Replica attaches locally found parts and sends info about them to Zookeeper. Parts present on a replica before metadata loss are not re-fetched from other ones if not being outdated (so replica restoration does not mean re-downloading all data over the network). !!! warning "Caveat" -Parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached. + Parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached. **Syntax** From 1ad1e62b47c5527a7ae8311470bf2fa09d66c0a9 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Aug 2021 04:02:48 +0300 Subject: [PATCH 043/220] Fix unit test --- src/Storages/tests/gtest_storage_log.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Storages/tests/gtest_storage_log.cpp b/src/Storages/tests/gtest_storage_log.cpp index 16902eafc98..b3ceef7e697 100644 --- a/src/Storages/tests/gtest_storage_log.cpp +++ b/src/Storages/tests/gtest_storage_log.cpp @@ -128,6 +128,7 @@ std::string readData(DB::StoragePtr & table, const DB::ContextPtr context) { ColumnWithTypeAndName col; col.type = std::make_shared(); + col.name = "a"; sample.insert(std::move(col)); } From 4b4cd59ea7f79668a5bef78136578b5ffd8a03d3 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 10 Aug 2021 00:48:44 +0300 Subject: [PATCH 044/220] Apply patch from @azat --- .../evaluateConstantExpression.cpp | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/evaluateConstantExpression.cpp b/src/Interpreters/evaluateConstantExpression.cpp index 6c08d481acf..c05118b7c6a 100644 --- a/src/Interpreters/evaluateConstantExpression.cpp +++ b/src/Interpreters/evaluateConstantExpression.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB { @@ -339,6 +340,7 @@ std::optional evaluateExpressionOverConstantCondition(const ASTPtr & nod if (const auto * fn = node->as()) { + std::unordered_map always_false_map; const auto dnf = analyzeFunction(fn, target_expr, limit); if (dnf.empty() || !limit) @@ -388,8 +390,19 @@ std::optional evaluateExpressionOverConstantCondition(const ASTPtr & nod Field prev_value = assert_cast(*prev.column).getField(); Field curr_value = assert_cast(*elem.column).getField(); - if (prev_value != curr_value) - return Blocks{}; + if (!always_false_map.count(elem.name)) + { + always_false_map[elem.name] = prev_value != curr_value; + } + else + { + auto & always_false = always_false_map[elem.name]; + /// If at least one of conjunct is not always false, we should preserve this. + if (always_false) + { + always_false = prev_value != curr_value; + } + } } } } @@ -417,6 +430,11 @@ std::optional evaluateExpressionOverConstantCondition(const ASTPtr & nod return {}; } } + + bool any_always_false = std::any_of(always_false_map.begin(), always_false_map.end(), [](const auto & v) { return v.second; }); + if (any_always_false) + return Blocks{}; + } else if (const auto * literal = node->as()) { From 669f89586f267d8068a159aa118114041571cafb Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Tue, 10 Aug 2021 14:25:19 -0300 Subject: [PATCH 045/220] Update graphitemergetree.md --- .../table-engines/mergetree-family/graphitemergetree.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md index 891d5227100..599e08bc7c3 100644 --- a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md @@ -38,9 +38,7 @@ CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] - Значение метрики. Тип данных: любой числовой. -- Версия метрики. Тип данных: любой числовой. - - ClickHouse сохраняет строки с последней версией или последнюю записанную строку, если версии совпадают. Другие строки удаляются при слиянии кусков данных. +- Версия метрики. Тип данных: любой числовой (ClickHouse сохраняет строки с последней версией или последнюю записанную строку, если версии совпадают. Другие строки удаляются при слиянии кусков данных). Имена этих столбцов должны быть заданы в конфигурации rollup. @@ -173,4 +171,4 @@ default !!! warning "Внимание" - Прореживание данных производится во время слияний. Обычно для старых партций слияния не запускаются, поэтому для прореживания надо иницировать незапланированное слияние используя [optimize](../../../sql-reference/statements/optimize/). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). + Прореживание данных производится во время слияний. Обычно для старых партций слияния не запускаются, поэтому для прореживания надо иницировать незапланированное слияние используя [optimize](../../../../sql-reference/statements/optimize/). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). From 4e62225a0be43565057e22fd54308708bf75daf4 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Tue, 10 Aug 2021 14:33:36 -0300 Subject: [PATCH 046/220] Update graphitemergetree.md --- .../table-engines/mergetree-family/graphitemergetree.md | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index 3ead798503d..30aa10ba38a 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -38,9 +38,7 @@ A table for the Graphite data should have the following columns for the followin - Value of the metric. Data type: any numeric. -- Version of the metric. Data type: any numeric. - - ClickHouse saves the rows with the highest version or the last written if versions are the same. Other rows are deleted during the merge of data parts. +- Version of the metric. Data type: any numeric (ClickHouse saves the rows with the highest version or the last written if versions are the same. Other rows are deleted during the merge of data parts). The names of these columns should be set in the rollup configuration. @@ -132,7 +130,7 @@ Fields for `pattern` and `default` sections: - `regexp`– A pattern for the metric name. - `age` – The minimum age of the data in seconds. - `precision`– How precisely to define the age of the data in seconds. Should be a divisor for 86400 (seconds in a day). -- `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. +- `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. Accepted functions: min / max / any / avg. The average is calculated imprecisely, like the average of the averages. ### Configuration Example {#configuration-example} @@ -169,4 +167,7 @@ Fields for `pattern` and `default` sections: ``` +!!! warning "Warning" + Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../../sql-reference/statements/optimize/). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). + [Original article](https://clickhouse.tech/docs/en/operations/table_engines/graphitemergetree/) From b324d85fbcf35a6be9be7a77e10683e33a9ad298 Mon Sep 17 00:00:00 2001 From: pdv-ru Date: Tue, 10 Aug 2021 21:04:42 +0300 Subject: [PATCH 047/220] DOCSUP-10607: small fix --- docs/en/sql-reference/statements/system.md | 6 +++--- docs/ru/sql-reference/functions/geo/h3.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 153db6963a0..3c3268f89c3 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -313,8 +313,8 @@ One may execute query after: Replica attaches locally found parts and sends info about them to Zookeeper. Parts present on a replica before metadata loss are not re-fetched from other ones if not being outdated (so replica restoration does not mean re-downloading all data over the network). -!!! warning "Caveat" - Parts in all states are moved to `detached/` folder. Parts active before data loss (Committed) are attached. +!!! warning "Warning" + Parts in all states are moved to `detached/` folder. Parts active before data loss (committed) are attached. **Syntax** @@ -330,7 +330,7 @@ SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_fami **Example** -Creating table on multiple servers. After the replica's root directory is lost, the table will will attach as readonly as metadata is missing. The last query need to execute on every replica. +Creating a table on multiple servers. After the replica's root directory is lost, the table will attach as read-only as metadata is missing. The last query needs to execute on every replica. ```sql CREATE TABLE test(n UInt32) diff --git a/docs/ru/sql-reference/functions/geo/h3.md b/docs/ru/sql-reference/functions/geo/h3.md index 2d33c6ba15a..3f58b034328 100644 --- a/docs/ru/sql-reference/functions/geo/h3.md +++ b/docs/ru/sql-reference/functions/geo/h3.md @@ -209,7 +209,7 @@ h3ToGeo(h3Index) **Возвращаемые значения** -- Набор из двух значений: `tuple(lon,lat)`. `lon` — долгота. [Float64](../../../sql-reference/data-types/float.md). `lat` — широта. [Float64](../../../sql-reference/data-types/float.md). +- кортеж из двух значений: `tuple(lon,lat)`, где `lon` — долгота [Float64](../../../sql-reference/data-types/float.md), `lat` — широта [Float64](../../../sql-reference/data-types/float.md). **Пример** From ca38b6b7f23c310fde220d6a8c8e8ec7c5b91fdf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Wed, 11 Aug 2021 06:06:20 +0300 Subject: [PATCH 048/220] Update test --- .../0_stateless/01950_kill_large_group_by_query.reference | 4 ++-- tests/queries/0_stateless/01950_kill_large_group_by_query.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/queries/0_stateless/01950_kill_large_group_by_query.reference b/tests/queries/0_stateless/01950_kill_large_group_by_query.reference index 1602d6587ad..f1df2658897 100644 --- a/tests/queries/0_stateless/01950_kill_large_group_by_query.reference +++ b/tests/queries/0_stateless/01950_kill_large_group_by_query.reference @@ -1,2 +1,2 @@ -finished test_01948_tcp_default default SELECT * FROM\n (\n SELECT a.name as n\n FROM\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) AS a,\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) as b\n GROUP BY n\n )\n LIMIT 20\n FORMAT Null -finished test_01948_http_default default SELECT * FROM\n (\n SELECT a.name as n\n FROM\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) AS a,\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) as b\n GROUP BY n\n )\n LIMIT 20\n FORMAT Null +finished test_01948_tcp_default default SELECT * FROM\n (\n SELECT a.name as n\n FROM\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) AS a,\n (\n SELECT \'Name\' as name2, number FROM system.numbers LIMIT 2000000\n ) as b\n GROUP BY n\n )\n LIMIT 20\n FORMAT Null +finished test_01948_http_default default SELECT * FROM\n (\n SELECT a.name as n\n FROM\n (\n SELECT \'Name\' as name, number FROM system.numbers LIMIT 2000000\n ) AS a,\n (\n SELECT \'Name\' as name2, number FROM system.numbers LIMIT 2000000\n ) as b\n GROUP BY n\n )\n LIMIT 20\n FORMAT Null diff --git a/tests/queries/0_stateless/01950_kill_large_group_by_query.sh b/tests/queries/0_stateless/01950_kill_large_group_by_query.sh index 465b923187e..0b369c7257e 100755 --- a/tests/queries/0_stateless/01950_kill_large_group_by_query.sh +++ b/tests/queries/0_stateless/01950_kill_large_group_by_query.sh @@ -23,7 +23,7 @@ $CLICKHOUSE_CLIENT --max_execution_time 10 --query_id "test_01948_tcp_$CLICKHOUS SELECT 'Name' as name, number FROM system.numbers LIMIT 2000000 ) AS a, ( - SELECT 'Name' as name, number FROM system.numbers LIMIT 2000000 + SELECT 'Name' as name2, number FROM system.numbers LIMIT 2000000 ) as b GROUP BY n ) @@ -44,7 +44,7 @@ ${CLICKHOUSE_CURL_COMMAND} -q --max-time 10 -sS "$CLICKHOUSE_URL&query_id=test_0 SELECT 'Name' as name, number FROM system.numbers LIMIT 2000000 ) AS a, ( - SELECT 'Name' as name, number FROM system.numbers LIMIT 2000000 + SELECT 'Name' as name2, number FROM system.numbers LIMIT 2000000 ) as b GROUP BY n ) From c35136a47b654da1a67a08c77db40a20e5d1f387 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 11 Aug 2021 20:08:51 +0300 Subject: [PATCH 049/220] Create zookeeper_log.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Выполнил описание новой системной таблицы zookeeper_log. --- .../operations/system-tables/zookeeper_log.md | 131 ++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 docs/en/operations/system-tables/zookeeper_log.md diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md new file mode 100644 index 00000000000..1d037382717 --- /dev/null +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -0,0 +1,131 @@ +# system.zookeeper_log {#system-zookeeper_log} + +The table does not exist if ZooKeeper is not configured. + +This table contains information about the parameters of the request to the ZooKeeper client and the response from it. + +For requests, only columns with request parameters are filled in, and the remaining columns are filled with default values (`0` or NULL). When the response arrives, the data from the response is added to the other columns. + +Columns with request parameters: + +- `type` ([Enum](../../sql-reference/data-types/enum.md)) — Event type in the ZooKeeper client. Can have one of the following values: + - `request` — The request has been sent. + - `response` — The response was received. + - `finalize` — The connection is lost, no response was received. +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the request was completed. +- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the request was completed. +- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the request. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — Host port. +- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — The session ID that the ZooKeeper server sets for each connection. +- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. Usually, it is just a sequential request number. It is the same for the request line and the paired `response`/`finalize` line. +- `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The request whether the [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches) has been installed. +- `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — The request or response type. +- `path` ([String](../../sql-reference/data-types/string.md)) — The path to the ZooKeeper node specified in the request (if the request requires specifying a path) or an empty string. +- `data` ([String](../../sql-reference/data-types/string.md)) — The data written to the ZooKeeper node (for the `SET` and `CREATE` requests — what the request wanted to write, for the response to the `GET` request — what was read) or an empty string. +- `is_ephemeral` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [ephemeral](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Ephemeral+Nodes). +- `is_sequential` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [sequential](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Sequence+Nodes+--+Unique+Naming). +- `version` ([Nullable(Int32)](../../sql-reference/data-types/nullable.md)) — The version of the ZooKeeper node that the request expects when executing (for `CHECK`, `SET`, `REMOVE` requests; `-1` if the request does not check the version) or NULL for other requests that do not support version checking. +- `requests_size` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of requests included in the "multi" request (this is a special request that consists of several consecutive ordinary requests and executes them atomically). All requests included in "multi" request will have the same `xid`. +- `request_idx` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of the request included in multi (for multi — `0`, then in order from `1`). + +Columns with request response parameters: + +- `zxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — ZooKeeper transaction id. The serial number issued by the ZooKeeper server in response to a successfully executed request (`0` if the request was not executed/returned an error/the client does not know whether the request was executed). +- `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — Error code. Can have one of the following values: + - `ZOK` — The response to the request was received. + - `ZCONNECTIONLOSS` — The connection was lost. + - `ZOPERATIONTIMEOUT` — The request execution timeout has expired. + - `ZSESSIONEXPIRED` — The session has expired. + - `NULL` — The request is completed. +- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: NULL. +- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: NULL. +- `path_created` ([String](../../sql-reference/data-types/string.md)) — The path to the created ZooKeeper node (for responses to the `CREATE` request), may differ from the `path` if the node is created as a sequential. +- `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that caused this ZooKeeper node to be created. +- `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that last modified this ZooKeeper node. +- `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The transaction id of the change that last modified childern of this ZooKeeper node. +- `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — The number of changes to the data of this ZooKeeper node. +- `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — The number of changes to the children of this ZooKeeper node. +- `stat_dataLength` ([Int32](../../sql-reference/data-types/int-uint.md)) — The length of the data field of this ZooKeeper node. +- `stat_numChildren` ([Int32](../../sql-reference/data-types/int-uint.md)) — The number of children of this ZooKeeper node. +- `children` ([Array(String)](../../sql-reference/data-types/array.md)) — The list of child ZooKeeper nodes (for responses to `LIST` request). + +**Example** + +Query: + +``` sql +SELECT * FROM system.zookeeper_log WHERE (session_id = '106662742089334927') AND (xid = '10858') FORMAT Vertical; +``` + +Result: + +``` text +Row 1: +────── +type: Request +event_date: 2021-08-09 +event_time: 2021-08-09 21:38:30.291792 +address: :: +port: 2181 +session_id: 106662742089334927 +xid: 10858 +has_watch: 1 +op_num: List +path: /clickhouse/task_queue/ddl +data: +is_ephemeral: 0 +is_sequential: 0 +version: ᴺᵁᴸᴸ +requests_size: 0 +request_idx: 0 +zxid: 0 +error: ᴺᵁᴸᴸ +watch_type: ᴺᵁᴸᴸ +watch_state: ᴺᵁᴸᴸ +path_created: +stat_czxid: 0 +stat_mzxid: 0 +stat_pzxid: 0 +stat_version: 0 +stat_cversion: 0 +stat_dataLength: 0 +stat_numChildren: 0 +children: [] + +Row 2: +────── +type: Response +event_date: 2021-08-09 +event_time: 2021-08-09 21:38:30.292086 +address: :: +port: 2181 +session_id: 106662742089334927 +xid: 10858 +has_watch: 1 +op_num: List +path: /clickhouse/task_queue/ddl +data: +is_ephemeral: 0 +is_sequential: 0 +version: ᴺᵁᴸᴸ +requests_size: 0 +request_idx: 0 +zxid: 16926267 +error: ZOK +watch_type: ᴺᵁᴸᴸ +watch_state: ᴺᵁᴸᴸ +path_created: +stat_czxid: 16925469 +stat_mzxid: 16925469 +stat_pzxid: 16926179 +stat_version: 0 +stat_cversion: 7 +stat_dataLength: 0 +stat_numChildren: 7 +children: ['query-0000000006','query-0000000005','query-0000000004','query-0000000003','query-0000000002','query-0000000001','query-0000000000'] +``` + +**See Also** + +- [ZooKeeper](../../operations/tips.md#zookeeper) +- [ZooKeeper guide](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html) From 3a32aa0dff9beeab376cdfafecdb985eaba5cdee Mon Sep 17 00:00:00 2001 From: Filatenkov Artur <58165623+FArthur-cmd@users.noreply.github.com> Date: Thu, 12 Aug 2021 16:03:35 +0300 Subject: [PATCH 050/220] Update table.md --- docs/en/sql-reference/statements/create/table.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index d09ff24efcd..c20981b6bbf 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -254,7 +254,6 @@ CREATE TABLE codec_example ENGINE = MergeTree() ``` - + ## Temporary Tables {#temporary-tables} ClickHouse supports temporary tables which have the following characteristics: From 49e211bead753e3d88a579cb42c09b653f179bd1 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Fri, 13 Aug 2021 16:30:28 +0000 Subject: [PATCH 051/220] add from infile syntax --- programs/client/Client.cpp | 23 ++++++++++++++++-- src/Parsers/ASTInsertQuery.cpp | 7 +++++- src/Parsers/ASTInsertQuery.h | 2 ++ src/Parsers/ParserInsertQuery.cpp | 15 +++++++++++- .../0_stateless/02009_from_infile.reference | 1 + .../queries/0_stateless/02009_from_infile.sh | 19 +++++++++++++++ tests/queries/0_stateless/test_infile.gz | Bin 0 -> 42 bytes 7 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02009_from_infile.reference create mode 100755 tests/queries/0_stateless/02009_from_infile.sh create mode 100644 tests/queries/0_stateless/test_infile.gz diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 14442167042..61a8168c6f4 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -2,6 +2,7 @@ #include "Common/MemoryTracker.h" #include "Columns/ColumnsNumber.h" #include "ConnectionParameters.h" +#include "IO/CompressionMethod.h" #include "QueryFuzzer.h" #include "Suggest.h" #include "TestHint.h" @@ -61,6 +62,7 @@ #include #include #include +#include #include #include #include @@ -1823,7 +1825,7 @@ private: void processInsertQuery() { const auto parsed_insert_query = parsed_query->as(); - if (!parsed_insert_query.data && (is_interactive || (!stdin_is_a_tty && std_in.eof()))) + if ((!parsed_insert_query.data && !parsed_insert_query.infile) && (is_interactive || (!stdin_is_a_tty && std_in.eof()))) throw Exception("No data to insert", ErrorCodes::NO_DATA_TO_INSERT); connection->sendQuery( @@ -1894,7 +1896,24 @@ private: if (!parsed_insert_query) return; - if (parsed_insert_query->data) + if (parsed_insert_query->infile) + { + const auto & in_file_node = parsed_insert_query->infile->as(); + const auto in_file = in_file_node.value.safeGet(); + + auto in_buffer = wrapReadBufferWithCompressionMethod(std::make_unique(in_file), chooseCompressionMethod(in_file, "")); + + try + { + sendDataFrom(*in_buffer, sample, columns_description); + } + catch (Exception & e) + { + e.addMessage("data for INSERT was parsed from query"); + throw; + } + } + else if (parsed_insert_query->data) { /// Send data contained in the query. ReadBufferFromMemory data_in(parsed_insert_query->data, parsed_insert_query->end - parsed_insert_query->data); diff --git a/src/Parsers/ASTInsertQuery.cpp b/src/Parsers/ASTInsertQuery.cpp index 8bfd3ccf1f2..39ae5f2a58a 100644 --- a/src/Parsers/ASTInsertQuery.cpp +++ b/src/Parsers/ASTInsertQuery.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -48,11 +49,15 @@ void ASTInsertQuery::formatImpl(const FormatSettings & settings, FormatState & s } else { + if (infile) + { + settings.ostr << (settings.hilite ? hilite_keyword : "") << " FROM INFILE " << (settings.hilite ? hilite_none : "") << infile->as().value.safeGet(); + } if (!format.empty()) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " FORMAT " << (settings.hilite ? hilite_none : "") << format; } - else + else if (!infile) { settings.ostr << (settings.hilite ? hilite_keyword : "") << " VALUES" << (settings.hilite ? hilite_none : ""); } diff --git a/src/Parsers/ASTInsertQuery.h b/src/Parsers/ASTInsertQuery.h index a454f46c3f1..e98fe79dedb 100644 --- a/src/Parsers/ASTInsertQuery.h +++ b/src/Parsers/ASTInsertQuery.h @@ -2,6 +2,7 @@ #include #include +#include "Parsers/IAST_fwd.h" namespace DB { @@ -16,6 +17,7 @@ public: ASTPtr columns; String format; ASTPtr select; + ASTPtr infile; ASTPtr watch; ASTPtr table_function; ASTPtr settings_ast; diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index 1f987edf13f..3252c4bc02c 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -11,6 +11,7 @@ #include #include #include +#include "Parsers/IAST_fwd.h" namespace DB @@ -25,6 +26,7 @@ namespace ErrorCodes bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_insert_into("INSERT INTO"); + ParserKeyword s_from_infile("FROM INFILE"); ParserKeyword s_table("TABLE"); ParserKeyword s_function("FUNCTION"); ParserToken s_dot(TokenType::Dot); @@ -39,9 +41,11 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ParserIdentifier name_p; ParserList columns_p(std::make_unique(), std::make_unique(TokenType::Comma), false); ParserFunction table_function_p{false}; + ParserStringLiteral infile_name_p; ASTPtr database; ASTPtr table; + ASTPtr infile; ASTPtr columns; ASTPtr format; ASTPtr select; @@ -86,11 +90,17 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) Pos before_values = pos; - /// VALUES or FORMAT or SELECT + + /// VALUES or FROM INFILE or FORMAT or SELECT if (s_values.ignore(pos, expected)) { data = pos->begin; } + else if (s_from_infile.ignore(pos, expected)) + { + if (!infile_name_p.parse(pos, infile, expected)) + return false; + } else if (s_format.ignore(pos, expected)) { if (!name_p.parse(pos, format, expected)) @@ -167,6 +177,9 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) auto query = std::make_shared(); node = query; + if (infile) + query->infile = infile; + if (table_function) { query->table_function = table_function; diff --git a/tests/queries/0_stateless/02009_from_infile.reference b/tests/queries/0_stateless/02009_from_infile.reference new file mode 100644 index 00000000000..e965047ad7c --- /dev/null +++ b/tests/queries/0_stateless/02009_from_infile.reference @@ -0,0 +1 @@ +Hello diff --git a/tests/queries/0_stateless/02009_from_infile.sh b/tests/queries/0_stateless/02009_from_infile.sh new file mode 100755 index 00000000000..6dee54d3963 --- /dev/null +++ b/tests/queries/0_stateless/02009_from_infile.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +set -e + +[ -e "${CLICKHOUSE_TMP}"/test_infile.gz ] && rm "${CLICKHOUSE_TMP}"/test_infile.gz +[ -e "${CLICKHOUSE_TMP}"/test_infile ] && rm "${CLICKHOUSE_TMP}"/test_infile + +echo "('Hello')" > "${CLICKHOUSE_TMP}"/test_infile + +gzip "${CLICKHOUSE_TMP}"/test_infile + +${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile;" +${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile (word String) ENGINE=Memory();" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz';" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile;" diff --git a/tests/queries/0_stateless/test_infile.gz b/tests/queries/0_stateless/test_infile.gz new file mode 100644 index 0000000000000000000000000000000000000000..feb3ac520687836a6f136474db87ac0148fd466d GIT binary patch literal 42 ycmb2|=HSSiBbLa(T#{N`5}%oumYI{va5><+$Jvu7!>>PKV(5D1wwQ~7fdK$Y3lI?i literal 0 HcmV?d00001 From 58b8e8f230f66a336391ef69692e207a88284f53 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Fri, 13 Aug 2021 16:55:03 +0000 Subject: [PATCH 052/220] correct commits --- docs/en/sql-reference/statements/create/table.md | 3 ++- tests/queries/0_stateless/test_infile.gz | Bin 42 -> 0 bytes 2 files changed, 2 insertions(+), 1 deletion(-) delete mode 100644 tests/queries/0_stateless/test_infile.gz diff --git a/docs/en/sql-reference/statements/create/table.md b/docs/en/sql-reference/statements/create/table.md index c20981b6bbf..d09ff24efcd 100644 --- a/docs/en/sql-reference/statements/create/table.md +++ b/docs/en/sql-reference/statements/create/table.md @@ -254,6 +254,7 @@ CREATE TABLE codec_example ENGINE = MergeTree() ``` + ## Temporary Tables {#temporary-tables} ClickHouse supports temporary tables which have the following characteristics: diff --git a/tests/queries/0_stateless/test_infile.gz b/tests/queries/0_stateless/test_infile.gz deleted file mode 100644 index feb3ac520687836a6f136474db87ac0148fd466d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 42 ycmb2|=HSSiBbLa(T#{N`5}%oumYI{va5><+$Jvu7!>>PKV(5D1wwQ~7fdK$Y3lI?i From 9ba9d39d42fa15c61ed826693a2868742e050252 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Sat, 14 Aug 2021 11:15:32 +0000 Subject: [PATCH 053/220] correct style --- programs/client/Client.cpp | 3 +-- src/Parsers/ParserInsertQuery.cpp | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index 61a8168c6f4..afc75300370 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -62,7 +62,6 @@ #include #include #include -#include #include #include #include @@ -1900,7 +1899,7 @@ private: { const auto & in_file_node = parsed_insert_query->infile->as(); const auto in_file = in_file_node.value.safeGet(); - + auto in_buffer = wrapReadBufferWithCompressionMethod(std::make_unique(in_file), chooseCompressionMethod(in_file, "")); try diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index 3252c4bc02c..9eb1cbfce02 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -90,7 +90,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) Pos before_values = pos; - + /// VALUES or FROM INFILE or FORMAT or SELECT if (s_values.ignore(pos, expected)) { From 60dccce8180f231845ab9eb2dfcc1f705157b5c6 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 15 Aug 2021 08:29:31 +0300 Subject: [PATCH 054/220] Remove trailing zeros from Decimal serialization #15794 --- src/Common/tests/gtest_wide_integer.cpp | 1 - src/IO/WriteHelpers.h | 58 ++++++++++++++----- .../02009_decimal_no_trailing_zeros.reference | 32 ++++++++++ .../02009_decimal_no_trailing_zeros.sql | 18 ++++++ 4 files changed, 95 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference create mode 100644 tests/queries/0_stateless/02009_decimal_no_trailing_zeros.sql diff --git a/src/Common/tests/gtest_wide_integer.cpp b/src/Common/tests/gtest_wide_integer.cpp index 982bbee804e..0cdbcb29c02 100644 --- a/src/Common/tests/gtest_wide_integer.cpp +++ b/src/Common/tests/gtest_wide_integer.cpp @@ -277,5 +277,4 @@ GTEST_TEST(WideInteger, DecimalFormatting) Int128 fractional = DecimalUtils::getFractionalPart(x, 2); EXPECT_EQ(fractional, 40); - EXPECT_EQ(decimalFractional(fractional, 2), "40"); } diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 556adbe2d6f..891a8d1f073 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -901,30 +901,63 @@ inline void writeText(const LocalDateTime & x, WriteBuffer & buf) { writeDateTim inline void writeText(const UUID & x, WriteBuffer & buf) { writeUUIDText(x, buf); } template -String decimalFractional(const T & x, UInt32 scale) +void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) { + /// If it's big integer, but the number of digits is small, + /// use the implementation for smaller integers for more efficient arithmetic. + if constexpr (std::is_same_v) { if (x <= std::numeric_limits::max()) - return decimalFractional(static_cast(x), scale); + { + writeDecimalFractional(static_cast(x), scale, ostr); + return; + } else if (x <= std::numeric_limits::max()) - return decimalFractional(static_cast(x), scale); + { + writeDecimalFractional(static_cast(x), scale, ostr); + return; + } else if (x <= std::numeric_limits::max()) - return decimalFractional(static_cast(x), scale); + { + writeDecimalFractional(static_cast(x), scale, ostr); + return; + } } else if constexpr (std::is_same_v) { if (x <= std::numeric_limits::max()) - return decimalFractional(static_cast(x), scale); + { + writeDecimalFractional(static_cast(x), scale, ostr); + return; + } else if (x <= std::numeric_limits::max()) - return decimalFractional(static_cast(x), scale); + { + writeDecimalFractional(static_cast(x), scale, ostr); + return; + } } - String str(scale, '0'); + constexpr size_t max_digits = std::numeric_limits::digits10; + assert(scale <= max_digits); + char buf[max_digits]; + memset(buf, '0', scale); + T value = x; - for (Int32 pos = scale - 1; pos >= 0; --pos, value /= 10) - str[pos] += static_cast(value % 10); - return str; + Int32 last_nonzero_pos = 0; + for (Int32 pos = scale - 1; pos >= 0; --pos) + { + auto remainder = value % 10; + value /= 10; + + if (remainder != 0 && last_nonzero_pos == 0) + last_nonzero_pos = pos; + + buf[pos] += static_cast(remainder); + } + + writeChar('.', ostr); + ostr.write(buf, last_nonzero_pos + 1); } template @@ -941,10 +974,9 @@ void writeText(Decimal x, UInt32 scale, WriteBuffer & ostr) if (scale) { - writeChar('.', ostr); part = DecimalUtils::getFractionalPart(x, scale); - String fractional = decimalFractional(part, scale); - ostr.write(fractional.data(), scale); + if (part) + writeDecimalFractional(part, scale, ostr); } } diff --git a/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference new file mode 100644 index 00000000000..d41682b62ce --- /dev/null +++ b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference @@ -0,0 +1,32 @@ +-- { echo } + +SELECT 1.123::Decimal64(1); +1.1 +SELECT 1.123::Decimal64(2); +1.12 +SELECT 1.123::Decimal64(3); +1.123 +SELECT 1.123::Decimal64(4); +1.123 +SELECT 1.123::Decimal64(5); +1.123 +SELECT 1.123::Decimal64(10); +1.123 +SELECT 1::Decimal64(0); +1 +SELECT 1::Decimal64(1); +1 +SELECT 1::Decimal64(10); +1 +SELECT 1.1234567::Decimal32(8); +1.1234567 +SELECT 1.1234567890::Decimal64(10); +1.123456789 +SELECT 1.1234567890::Decimal128(10); +1.123456789 +SELECT 1.1234567890::Decimal256(10); +1.123456789 +SELECT 1.123456789012345678901::Decimal256(20); +1.1234567890123456789 +SELECT 1.123456789012345678901::Decimal256(22); +1.123456789012345678901 diff --git a/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.sql b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.sql new file mode 100644 index 00000000000..556e355e7d8 --- /dev/null +++ b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.sql @@ -0,0 +1,18 @@ +-- { echo } + +SELECT 1.123::Decimal64(1); +SELECT 1.123::Decimal64(2); +SELECT 1.123::Decimal64(3); +SELECT 1.123::Decimal64(4); +SELECT 1.123::Decimal64(5); +SELECT 1.123::Decimal64(10); +SELECT 1::Decimal64(0); +SELECT 1::Decimal64(1); +SELECT 1::Decimal64(10); + +SELECT 1.1234567::Decimal32(8); +SELECT 1.1234567890::Decimal64(10); +SELECT 1.1234567890::Decimal128(10); +SELECT 1.1234567890::Decimal256(10); +SELECT 1.123456789012345678901::Decimal256(20); +SELECT 1.123456789012345678901::Decimal256(22); From e2a17c08b7741ac0bdbacd6e3cf6b87831114dfa Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 15 Aug 2021 09:09:40 +0300 Subject: [PATCH 055/220] Temporary disable one test case --- tests/queries/0_stateless/00597_push_down_predicate_long.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql index 2e3357241ad..412b8b7852c 100644 --- a/tests/queries/0_stateless/00597_push_down_predicate_long.sql +++ b/tests/queries/0_stateless/00597_push_down_predicate_long.sql @@ -8,7 +8,8 @@ DROP TABLE IF EXISTS test_view_00597; CREATE TABLE test_00597(date Date, id Int8, name String, value Int64) ENGINE = MergeTree(date, (id, date), 8192); CREATE VIEW test_view_00597 AS SELECT * FROM test_00597; -SELECT * FROM (SELECT floor(floor(1, floor(NULL), id = 257), floor(floor(floor(floor(NULL), '10485.76', '9223372036854775807', NULL), floor(10, floor(65535, NULL), 100.0000991821289), NULL)), '2.56'), b.* FROM (SELECT floor(floor(floor(floor(NULL), 1000.0001220703125))), * FROM test_00597) AS b) WHERE id = 257; +-- TODO: This query should execute successfully: +SELECT * FROM (SELECT floor(floor(1, floor(NULL), id = 257), floor(floor(floor(floor(NULL), '10485.76', '9223372036854775807', NULL), floor(10, floor(65535, NULL), 100.0000991821289), NULL)), '2.56'), b.* FROM (SELECT floor(floor(floor(floor(NULL), 1000.0001220703125))), * FROM test_00597) AS b) WHERE id = 257; -- { serverError 96 } INSERT INTO test_00597 VALUES('2000-01-01', 1, 'test string 1', 1); INSERT INTO test_00597 VALUES('2000-01-01', 2, 'test string 2', 2); From f5397c4430b77d0974c6a8632e0656f757b043c5 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 15 Aug 2021 10:53:46 +0300 Subject: [PATCH 056/220] Update tests --- .../0_stateless/00027_argMinMax.reference | 2 +- .../0_stateless/00502_sum_map.reference | 4 +- .../00700_decimal_aggregates.reference | 130 +- .../00700_decimal_arithm.reference | 46 +- .../00700_decimal_array_functions.reference | 30 +- .../00700_decimal_bounds.reference | 86 +- .../0_stateless/00700_decimal_casts.reference | 218 +- .../00700_decimal_casts_2.reference | 48 +- .../00700_decimal_compare.reference | 26 +- .../00700_decimal_complex_types.reference | 110 +- .../00700_decimal_defaults.reference | 16 +- .../00700_decimal_empty_aggregates.reference | 86 +- .../00700_decimal_formats.reference | 78 +- .../00700_decimal_gathers.reference | 24 +- .../00700_decimal_in_keys.reference | 44 +- .../0_stateless/00700_decimal_math.reference | 54 +- .../0_stateless/00700_decimal_null.reference | 10 +- .../0_stateless/00700_decimal_round.reference | 150 +- .../00700_to_decimal_or_something.reference | 12 +- ...00732_decimal_summing_merge_tree.reference | 4 +- .../00737_decimal_group_by.reference | 22 +- ...4_test_custom_compression_codecs.reference | 2 +- .../0_stateless/00805_round_down.reference | 60 +- .../0_stateless/00837_minmax_index.reference | 8 +- .../0_stateless/00838_unique_index.reference | 24 +- .../00861_decimal_quoted_csv.reference | 8 +- .../0_stateless/00862_decimal_in.reference | 36 +- ...0_decimal_group_array_crash_3783.reference | 16 +- .../00927_asof_join_other_types.reference | 18 +- .../0_stateless/00950_dict_get.reference | 30 +- .../0_stateless/00975_values_list.reference | 2 +- .../00979_toFloat_monotonicity.reference | 4 +- .../00980_crash_nullable_decimal.reference | 14 +- .../01018_empty_aggregation_filling.reference | 2 +- ...01055_minmax_index_compact_parts.reference | 8 +- .../01087_storage_generate.reference | 26 +- .../01087_table_function_generate.reference | 6 +- .../01095_tpch_like_smoke.reference | 6 +- .../01178_int_field_to_decimal.reference | 4 +- ...ialized_view_different_structure.reference | 2 +- .../01186_conversion_to_nullable.reference | 6 +- .../01231_markdown_format.reference | 4 +- .../01260_ubsan_decimal_parse.reference | 2 +- .../01280_min_map_max_map.reference | 4 +- ...mal_cut_extra_digits_after_point.reference | 8 +- .../01412_group_array_moving_shard.reference | 16 +- ...imal_parse_big_negative_exponent.reference | 2 +- ...01425_default_value_of_type_name.reference | 2 +- .../01440_big_int_exotic_casts.reference | 364 +- .../0_stateless/01459_decimal_casts.reference | 18 +- .../01474_decimal_scale_bug.reference | 36 +- ...1501_cache_dictionary_all_fields.reference | 12 +- ...01518_nullable_aggregate_states2.reference | 3636 ++++++++--------- .../01556_accurate_cast_or_null.reference | 2 +- .../01592_window_functions.reference | 148 +- .../0_stateless/01601_accurate_cast.reference | 2 +- .../01602_array_aggregation.reference | 18 +- .../0_stateless/01622_byte_size.reference | 8 +- ...ing_type_convert_to_decimal_type.reference | 2 +- .../01676_reinterpret_as.reference | 10 +- .../01711_decimal_multiplication.reference | 8 +- .../01721_dictionary_decimal_p_s.reference | 10 +- ...01804_dictionary_decimal256_type.reference | 12 +- .../0_stateless/01852_cast_operator.reference | 4 +- .../01852_cast_operator_2.reference | 4 +- ...cache_dictionary_decimal256_type.reference | 2 +- .../01913_if_int_decimal.reference | 6 +- 67 files changed, 2911 insertions(+), 2911 deletions(-) diff --git a/tests/queries/0_stateless/00027_argMinMax.reference b/tests/queries/0_stateless/00027_argMinMax.reference index 101e8c16044..c92140c0f33 100644 --- a/tests/queries/0_stateless/00027_argMinMax.reference +++ b/tests/queries/0_stateless/00027_argMinMax.reference @@ -1,5 +1,5 @@ 0 9 0 9 1970-01-01 1970-01-10 -0.00 9.00 +0 9 4 1 diff --git a/tests/queries/0_stateless/00502_sum_map.reference b/tests/queries/0_stateless/00502_sum_map.reference index c38fb2ec7d6..efd5a5534d4 100644 --- a/tests/queries/0_stateless/00502_sum_map.reference +++ b/tests/queries/0_stateless/00502_sum_map.reference @@ -22,5 +22,5 @@ ([1.01],[1]) (['a','b'],[1,2]) (['a','ab','abc'],[3,2,1]) -([1,2,3,4,5,6,7,8],[1.00000,2.00000,6.00000,8.00000,10.00000,12.00000,7.00000,8.00000]) -([1,2,3,4,5,6,7,8],[1.00000,2.00000,6.00000,8.00000,10.00000,12.00000,7.00000,8.00000]) +([1,2,3,4,5,6,7,8],[1,2,6,8,10,12,7,8]) +([1,2,3,4,5,6,7,8],[1,2,6,8,10,12,7,8]) diff --git a/tests/queries/0_stateless/00700_decimal_aggregates.reference b/tests/queries/0_stateless/00700_decimal_aggregates.reference index 251445675a2..159091d867e 100644 --- a/tests/queries/0_stateless/00700_decimal_aggregates.reference +++ b/tests/queries/0_stateless/00700_decimal_aggregates.reference @@ -1,72 +1,72 @@ 101 101 101 -[-50.0000,50.0000] [-16.66666666,16.66666666] [-10.00000000,10.00000000] -0.0000 0.00000000 0.00000000 0.0000 0.00000000 0.00000000 -1275.0000 424.99999983 255.00000000 1275.0000 424.99999983 255.00000000 --1275.0000 -424.99999983 -255.00000000 -1275.0000 -424.99999983 -255.00000000 -101.0000 101.00000000 101.00000000 101.0000 101.00000000 101.00000000 --101.0000 -101.00000000 -101.00000000 -101.0000 -101.00000000 -101.00000000 +[-50,50] [-16.66666666,16.66666666] [-10,10] +0 0 0 0 0 0 +1275 424.99999983 255 1275 424.99999983 255 +-1275 -424.99999983 -255 -1275 -424.99999983 -255 +101 101 101 101 101 101 +-101 -101 -101 -101 -101 -101 (101,101,101) (101,101,101) (101,101,101) (101,101,101) (102,100,101) 5 5 5 10 10 10 --50.0000 -50.0000 -16.66666666 -16.66666666 -10.00000000 -10.00000000 -1.0000 1.0000 0.33333333 0.33333333 0.20000000 0.20000000 -50.0000 50.0000 16.66666666 16.66666666 10.00000000 10.00000000 --1.0000 -1.0000 -0.33333333 -0.33333333 -0.20000000 -0.20000000 -0.0000 0.00000000 0.00000000 Decimal(38, 8) --25.5000 -8.49999999 -5.10000000 Decimal(38, 8) -0.0000 0.00000000 0.00000000 -10.0000 3.33333333 2.00000000 -20.0000 6.66666666 4.00000000 -30.0000 10.00000000 6.00000000 -40.0000 13.33333333 8.00000000 -50.0000 16.66666666 10.00000000 -[-50.0000,-40.0000,-30.0000,-20.0000,-10.0000,0.0000,10.0000,20.0000,30.0000,40.0000,50.0000] -[-16.66666666,-13.33333333,-10.00000000,-6.66666666,-3.33333333,0.00000000,3.33333333,6.66666666,10.00000000,13.33333333,16.66666666] -[-10.00000000,-8.00000000,-6.00000000,-4.00000000,-2.00000000,0.00000000,2.00000000,4.00000000,6.00000000,8.00000000,10.00000000] -0.0000 0.00000000 0.00000000 Decimal(38, 8) --25.0000 -8.33333333 -5.00000000 Decimal(38, 8) -0.0000 0.00000000 0.00000000 -10.0000 3.33333333 2.00000000 -20.0000 6.66666666 4.00000000 -30.0000 10.00000000 6.00000000 -40.0000 13.33333333 8.00000000 -50.0000 16.66666666 10.00000000 -[-50.0000,-40.0000,-30.0000,-20.0000,-10.0000,0.0000,10.0000,20.0000,30.0000,40.0000,50.0000] -[-16.66666666,-13.33333333,-10.00000000,-6.66666666,-3.33333333,0.00000000,3.33333333,6.66666666,10.00000000,13.33333333,16.66666666] -[-10.00000000,-8.00000000,-6.00000000,-4.00000000,-2.00000000,0.00000000,2.00000000,4.00000000,6.00000000,8.00000000,10.00000000] -0.0000 0.00000000 0.00000000 Decimal(38, 8) --26.0000 -8.66666666 -5.20000000 Decimal(38, 8) -0.0000 0.00000000 0.00000000 -10.0000 3.33333333 2.00000000 -20.0000 6.66666666 4.00000000 -30.0000 10.00000000 6.00000000 -40.0000 13.33333333 8.00000000 -50.0000 16.66666666 10.00000000 -[-50.0000,-40.0000,-30.0000,-20.0000,-10.0000,0.0000,10.0000,20.0000,30.0000,40.0000,50.0000] -[-16.66666666,-13.33333333,-10.00000000,-6.66666666,-3.33333333,0.00000000,3.33333333,6.66666666,10.00000000,13.33333333,16.66666666] -[-10.00000000,-8.00000000,-6.00000000,-4.00000000,-2.00000000,0.00000000,2.00000000,4.00000000,6.00000000,8.00000000,10.00000000] -0.0000 0.00000000 0.00000000 Decimal(38, 8) --25.0000 -8.33333333 -5.00000000 Decimal(38, 8) -0.0000 0.00000000 0.00000000 -10.0000 3.33333333 2.00000000 -20.0000 6.66666666 4.00000000 -30.0000 10.00000000 6.00000000 -40.0000 13.33333333 8.00000000 -50.0000 16.66666666 10.00000000 -[-50.0000,-40.0000,-30.0000,-20.0000,-10.0000,0.0000,10.0000,20.0000,30.0000,40.0000,50.0000] -[-16.66666666,-13.33333333,-10.00000000,-6.66666666,-3.33333333,0.00000000,3.33333333,6.66666666,10.00000000,13.33333333,16.66666666] -[-10.00000000,-8.00000000,-6.00000000,-4.00000000,-2.00000000,0.00000000,2.00000000,4.00000000,6.00000000,8.00000000,10.00000000] -0.0000 0.00000000 0.00000000 Decimal(38, 8) --26.0000 -8.66666666 -5.20000000 Decimal(38, 8) -0.0000 0.00000000 0.00000000 -10.0000 3.33333333 2.00000000 -20.0000 6.66666666 4.00000000 -30.0000 10.00000000 6.00000000 -40.0000 13.33333333 8.00000000 -50.0000 16.66666666 10.00000000 -[-50.0000,-40.0000,-30.0000,-20.0000,-10.0000,0.0000,10.0000,20.0000,30.0000,40.0000,50.0000] -[-16.66666666,-13.33333333,-10.00000000,-6.66666666,-3.33333333,0.00000000,3.33333333,6.66666666,10.00000000,13.33333333,16.66666666] -[-10.00000000,-8.00000000,-6.00000000,-4.00000000,-2.00000000,0.00000000,2.00000000,4.00000000,6.00000000,8.00000000,10.00000000] +-50 -50 -16.66666666 -16.66666666 -10 -10 +1 1 0.33333333 0.33333333 0.2 0.2 +50 50 16.66666666 16.66666666 10 10 +-1 -1 -0.33333333 -0.33333333 -0.2 -0.2 +0 0 0 Decimal(38, 8) +-25.5 -8.49999999 -5.1 Decimal(38, 8) +0 0 0 +10 3.33333333 2 +20 6.66666666 4 +30 10 6 +40 13.33333333 8 +50 16.66666666 10 +[-50,-40,-30,-20,-10,0,10,20,30,40,50] +[-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666] +[-10,-8,-6,-4,-2,0,2,4,6,8,10] +0 0 0 Decimal(38, 8) +-25 -8.33333333 -5 Decimal(38, 8) +0 0 0 +10 3.33333333 2 +20 6.66666666 4 +30 10 6 +40 13.33333333 8 +50 16.66666666 10 +[-50,-40,-30,-20,-10,0,10,20,30,40,50] +[-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666] +[-10,-8,-6,-4,-2,0,2,4,6,8,10] +0 0 0 Decimal(38, 8) +-26 -8.66666666 -5.2 Decimal(38, 8) +0 0 0 +10 3.33333333 2 +20 6.66666666 4 +30 10 6 +40 13.33333333 8 +50 16.66666666 10 +[-50,-40,-30,-20,-10,0,10,20,30,40,50] +[-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666] +[-10,-8,-6,-4,-2,0,2,4,6,8,10] +0 0 0 Decimal(38, 8) +-25 -8.33333333 -5 Decimal(38, 8) +0 0 0 +10 3.33333333 2 +20 6.66666666 4 +30 10 6 +40 13.33333333 8 +50 16.66666666 10 +[-50,-40,-30,-20,-10,0,10,20,30,40,50] +[-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666] +[-10,-8,-6,-4,-2,0,2,4,6,8,10] +0 0 0 Decimal(38, 8) +-26 -8.66666666 -5.2 Decimal(38, 8) +0 0 0 +10 3.33333333 2 +20 6.66666666 4 +30 10 6 +40 13.33333333 8 +50 16.66666666 10 +[-50,-40,-30,-20,-10,0,10,20,30,40,50] +[-16.66666666,-13.33333333,-10,-6.66666666,-3.33333333,0,3.33333333,6.66666666,10,13.33333333,16.66666666] +[-10,-8,-6,-4,-2,0,2,4,6,8,10] 850 94.44444438684269 34 Float64 Float64 Float64 850 94.4444443868427 34.00000000000001 858.5 95.38888883071111 34.34 Float64 Float64 Float64 diff --git a/tests/queries/0_stateless/00700_decimal_arithm.reference b/tests/queries/0_stateless/00700_decimal_arithm.reference index 9de0d4cbf9a..a41ef5b0557 100644 --- a/tests/queries/0_stateless/00700_decimal_arithm.reference +++ b/tests/queries/0_stateless/00700_decimal_arithm.reference @@ -1,37 +1,37 @@ 84 0 1764 1 1 1 84 0 1764 1 1 1 84 0 1764 1 1 1 -84.840 0.000 1799.456400 1.000 1.000 1.000 -84.840000000 0.000000000 -84.840000000000000000 0.000000000000000000 98.044565395307682683126962841158942720 1.000000000000000000 1.000000000000000000 1.000000000000000000 -84.840000000000000000 0.000000000000000000 -84.84 0.00 1799.4564 1.00 1.00 1.00 +84.84 0 1799.4564 1 1 1 +84.84 0 +84.84 0 98.04456539530768268312696284115894272 1 1 1 +84.84 0 +84.84 0 1799.4564 1 1 1 63 21 -42 882 -882 2 0 2 0 63 21 -42 882 -882 2 0 2 0 63 21 -42 882 -882 2 0 2 0 -1.00305798474369219219752355409390731264 -0.16305798474369219219752355409390731264 1.49059173023461586584365185794205286400 -1.38847100762815390390123822295304634368 1.38847100762815390390123822295304634368 0.02000000000000000000000000000000000000 0.00500000000000000000000000000000000000 -63.420 21.420 -41.580 890.820 -890.820 2.020 0.505 2.020 0.505 -63.420000000 21.420000000 -41.580000000 890.820000000 -890.820000000 2.020000000 0.505000000 2.020000000 0.505000000 -63.420000000000000000 21.420000000000000000 -41.580000000000000000 890.820000000000000000 -890.820000000000000000 2.020000000000000000 0.505000000000000000 2.020000000000000000 0.505000000000000000 -63.42 21.42 -41.58 890.82 -890.82 2.02 0.50 2.02 0.50 +1.00305798474369219219752355409390731264 -0.16305798474369219219752355409390731264 1.490591730234615865843651857942052864 -1.38847100762815390390123822295304634368 1.38847100762815390390123822295304634368 0.02 0.005 +63.42 21.42 -41.58 890.82 -890.82 2.02 0.505 2.02 0.505 +63.42 21.42 -41.58 890.82 -890.82 2.02 0.505 2.02 0.505 +63.42 21.42 -41.58 890.82 -890.82 2.02 0.505 2.02 0.505 +63.42 21.42 -41.58 890.82 -890.82 2.02 0.5 2.02 0.5 63 -21 42 882 -882 0 2 0 2 63 -21 42 882 -882 0 2 0 2 63 -21 42 882 -882 0 2 0 2 -1.00305798474369219219752355409390731264 0.16305798474369219219752355409390731264 -1.49059173023461586584365185794205286400 -1.38847100762815390390123822295304634368 1.38847100762815390390123822295304634368 -0.00000000000000000000000000000000000001 0.00000000000000000000000000000000000001 -63.420 -21.420 41.580 890.820 -890.820 0.495 1.980 0.495 1.980 -63.420000000 -21.420000000 41.580000000 890.820000000 -890.820000000 -63.420000000000000000 -21.420000000000000000 41.580000000000000000 890.820000000000000000 -890.820000000000000000 0.495049504950495049 1.980198019801980198 0.495049504950495049 1.980198019801980198 +1.00305798474369219219752355409390731264 0.16305798474369219219752355409390731264 -1.490591730234615865843651857942052864 -1.38847100762815390390123822295304634368 1.38847100762815390390123822295304634368 -0.00000000000000000000000000000000000001 0.00000000000000000000000000000000000001 +63.42 -21.42 41.58 890.82 -890.82 0.495 1.98 0.495 1.98 +63.42 -21.42 41.58 890.82 -890.82 +63.42 -21.42 41.58 890.82 -890.82 0.495049504950495049 1.980198019801980198 0.495049504950495049 1.980198019801980198 63.42 -21.42 41.58 890.82 -890.82 0.49 1.98 0.49 1.98 --42 42 42 42 0.420000000 0.420000000000000000 0.42000000000000000000000000000000000000 42.420 42.420000000 42.42 -0 0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.000 0.000000000 0.00 -42 -42 -42 -42 -0.420000000 -0.420000000000000000 -0.42000000000000000000000000000000000000 -42.420 -42.420000000 -42.42 -42 42 42 0.420000000 0.420000000000000000 0.42000000000000000000000000000000000000 42.420 42.420000000 42.42 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.000 0.000000000 0.00 -42 42 42 0.420000000 0.420000000000000000 0.42000000000000000000000000000000000000 42.420 42.420000000 42.42 +-42 42 42 42 0.42 0.42 0.42 42.42 42.42 42.42 +0 0 0 0 0 0 0 0 0 0 +42 -42 -42 -42 -0.42 -0.42 -0.42 -42.42 -42.42 -42.42 +42 42 42 0.42 0.42 0.42 42.42 42.42 42.42 +0 0 0 0 0 0 0 0 0 +42 42 42 0.42 0.42 0.42 42.42 42.42 42.42 1 1 1 1 1 0 1 0 1 0 1 0 -0.0000 \N \N -0.00000000 \N \N -0.000000000000000000 \N \N +0 \N \N +0 \N \N +0 \N \N diff --git a/tests/queries/0_stateless/00700_decimal_array_functions.reference b/tests/queries/0_stateless/00700_decimal_array_functions.reference index 969a8dd2f18..ae872b7a347 100644 --- a/tests/queries/0_stateless/00700_decimal_array_functions.reference +++ b/tests/queries/0_stateless/00700_decimal_array_functions.reference @@ -1,20 +1,20 @@ -[0.0000,1.0000] Array(Decimal(9, 4)) -[0.00000000,1.00000000] Array(Decimal(18, 8)) -[0.00000000,1.00000000] Array(Decimal(38, 8)) +[0,1] Array(Decimal(9, 4)) +[0,1] Array(Decimal(18, 8)) +[0,1] Array(Decimal(38, 8)) - -1.0000 Decimal(38, 4) -1.00000000 Decimal(38, 8) -1.00000000 Decimal(38, 8) +1 Decimal(38, 4) +1 Decimal(38, 8) +1 Decimal(38, 8) - -[1.0000,2.0000] Array(Decimal(38, 4)) -[1.00000000,2.00000000] Array(Decimal(38, 8)) -[1.00000000,2.00000000] Array(Decimal(38, 8)) +[1,2] Array(Decimal(38, 4)) +[1,2] Array(Decimal(38, 8)) +[1,2] Array(Decimal(38, 8)) - -[1.0000,2.0000] Array(Decimal(38, 4)) -[1.00000000,2.00000000] Array(Decimal(38, 8)) -[1.00000000,2.00000000] Array(Decimal(38, 8)) +[1,2] Array(Decimal(38, 4)) +[1,2] Array(Decimal(38, 8)) +[1,2] Array(Decimal(38, 8)) - -[1.0000] Array(Decimal(9, 4)) -[1.00000000] Array(Decimal(18, 8)) -[1.00000000] Array(Decimal(38, 8)) +[1] Array(Decimal(9, 4)) +[1] Array(Decimal(18, 8)) +[1] Array(Decimal(38, 8)) - diff --git a/tests/queries/0_stateless/00700_decimal_bounds.reference b/tests/queries/0_stateless/00700_decimal_bounds.reference index 3f25fccc942..86688ea0546 100644 --- a/tests/queries/0_stateless/00700_decimal_bounds.reference +++ b/tests/queries/0_stateless/00700_decimal_bounds.reference @@ -1,43 +1,43 @@ --999999999 -999999999999999999 0 -0.999999999 0.000000000000000000 0.00000000000000000000000000000000000000 -9999.99999 0.000000000 0.000000000000000000 0 --900000000 -900000000000000000 -90000000000000000000000000000000000000 -0.000000009 -0.000000000000000009 -0.00000000000000000000000000000000000009 0.00000 0.000000000 0.000000000000000000 0 --1 -1 -1 -0.000000001 0.000000000000000000 0.00000000000000000000000000000000000000 -0.00001 -0.000000001 0.000000000000000000 -1 -0 0 -99999999999999999999999999999999999999 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 -0.999999999999999999 0.00000000000000000000000000000000000000 0.00000 -999999999.999999999 0.000000000000000000 0 -0 0 0 0.000000000 -0.000000000000000001 -0.00000000000000000000000000000000000001 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 -0.99999999999999999999999999999999999999 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 -99999999999999999999.999999999999999999 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 -0.000000000000000001 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000001 0 -0 0 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 99999999999999999999.999999999999999999 0 -0 0 0 0.000000000 0.000000000000000000 0.99999999999999999999999999999999999999 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.000000000000000001 0.00000000000000000000000000000000000001 0.00000 0.000000000 0.000000000000000000 0 -0 0 0 0.000000000 0.999999999999999999 0.00000000000000000000000000000000000000 0.00000 999999999.999999999 0.000000000000000000 0 -0 0 99999999999999999999999999999999999999 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.00000 0.000000000 0.000000000000000000 0 -1 1 1 0.000000001 0.000000000000000000 0.00000000000000000000000000000000000000 0.00001 0.000000001 0.000000000000000000 1 -42 42 0 0.000000000 0.000000000000000000 0.00000000000000000000000000000000000000 0.99999 0.000000000 0.000000000000000000 0 -900000000 900000000000000000 90000000000000000000000000000000000000 0.000000009 0.000000000000000009 0.00000000000000000000000000000000000009 0.00000 0.000000000 0.000000000000000000 0 -999999999 999999999999999999 0 0.999999999 0.000000000000000000 0.00000000000000000000000000000000000000 9999.99999 0.000000000 0.000000000000000000 0 +-999999999 -999999999999999999 0 -0.999999999 0 0 -9999.99999 0 0 0 +-900000000 -900000000000000000 -90000000000000000000000000000000000000 -0.000000009 -0.000000000000000009 -0.00000000000000000000000000000000000009 0 0 0 0 +-1 -1 -1 -0.000000001 0 0 -0.00001 -0.000000001 0 -1 +0 0 -99999999999999999999999999999999999999 0 0 0 0 0 0 0 +0 0 0 0 -0.999999999999999999 0 0 -999999999.999999999 0 0 +0 0 0 0 -0.000000000000000001 -0.00000000000000000000000000000000000001 0 0 0 0 +0 0 0 0 0 -0.99999999999999999999999999999999999999 0 0 0 0 +0 0 0 0 0 0 0 0 -99999999999999999999.999999999999999999 0 +0 0 0 0 0 0 0 0 -0.000000000000000001 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0 0 +0 0 0 0 0 0 0 0 0.000000000000000001 0 +0 0 0 0 0 0 0 0 99999999999999999999.999999999999999999 0 +0 0 0 0 0 0.99999999999999999999999999999999999999 0 0 0 0 +0 0 0 0 0.000000000000000001 0.00000000000000000000000000000000000001 0 0 0 0 +0 0 0 0 0.999999999999999999 0 0 999999999.999999999 0 0 +0 0 99999999999999999999999999999999999999 0 0 0 0 0 0 0 +1 1 1 0.000000001 0 0 0.00001 0.000000001 0 1 +42 42 0 0 0 0 0.99999 0 0 0 +900000000 900000000000000000 90000000000000000000000000000000000000 0.000000009 0.000000000000000009 0.00000000000000000000000000000000000009 0 0 0 0 +999999999 999999999999999999 0 0.999999999 0 0 9999.99999 0 0 0 diff --git a/tests/queries/0_stateless/00700_decimal_casts.reference b/tests/queries/0_stateless/00700_decimal_casts.reference index 99d8b949398..9f469f2907e 100644 --- a/tests/queries/0_stateless/00700_decimal_casts.reference +++ b/tests/queries/0_stateless/00700_decimal_casts.reference @@ -1,30 +1,30 @@ -1.1 1.10 1.10000000 +1.1 1.1 1.1 1 -1 1.1 1.10 1.10000000 +1 1.1 1.1 1.1 0.1 0 -0.1 0 0.1 0 -0.1 0 0.1 0 -0.1 0 -0.0000000001 0.000000000 --0.0000000001 0.000000000 -0.0000000000000000001 0.000000000000000000 --0.0000000000000000001 0.000000000000000000 -0.000000000000000000000000000000000000001 0.00000000000000000000000000000000000000 --0.000000000000000000000000000000000000001 0.00000000000000000000000000000000000000 +0.0000000001 0 +-0.0000000001 0 +0.0000000000000000001 0 +-0.0000000000000000001 0 +0.000000000000000000000000000000000000001 0 +-0.000000000000000000000000000000000000001 0 1e-1 0 -1e-1 0 1e-1 0 -1e-1 0 1e-1 0 -1e-1 0 -1e-10 0.000000000 --1e-10 0.000000000 -1e-19 0.000000000000000000 --1e-19 0.000000000000000000 -1e-39 0.00000000000000000000000000000000000000 --1e-39 0.00000000000000000000000000000000000000 +1e-10 0 +-1e-10 0 +1e-19 0 +-1e-19 0 +1e-39 0 +-1e-39 0 9999999 9999999 -9999999 9999999 -9999999 999999.9 999999.9 -999999.9 999999.9 -999999.9 99999.99 99999.99 -99999.99 99999.99 -99999.99 @@ -33,8 +33,8 @@ 99.99999 99.99999 -99.99999 99.99999 -99.99999 9.999999 9.999999 -9.999999 9.999999 -9.999999 0.9999999 0.9999999 -0.9999999 0.9999999 -0.9999999 -10 10.00000000 -10.00000000 10.00000000 -10.00000000 -1 1.000000000 -1.000000000 1.000000000 -1.000000000 +10 10 -10 10 -10 +1 1 -1 1 -1 999999999 999999999 -999999999 999999999 -999999999 99999999.9 99999999.9 -99999999.9 99999999.9 -99999999.9 9999999.99 9999999.99 -9999999.99 9999999.99 -9999999.99 @@ -45,111 +45,111 @@ 99.9999999 99.9999999 -99.9999999 99.9999999 -99.9999999 9.99999999 9.99999998 -9.99999998 9.99999998 -9.99999998 0.999999999 0.999999999 -0.999999999 0.999999999 -0.999999999 -1000000000 1000000000.000000000 -1000000000.000000000 -100000000 100000000.0000000000 -100000000.0000000000 -10000000 10000000.00000000000 -10000000.00000000000 -1000000 1000000.000000000000 -1000000.000000000000 -100000 100000.0000000000000 -100000.0000000000000 -10000 10000.00000000000000 -10000.00000000000000 -1000 1000.000000000000000 -1000.000000000000000 -100 100.0000000000000000 -100.0000000000000000 -10 10.00000000000000000 -10.00000000000000000 -1 1.000000000000000000 -1.000000000000000000 +1000000000 1000000000 -1000000000 +100000000 100000000 -100000000 +10000000 10000000 -10000000 +1000000 1000000 -1000000 +100000 100000 -100000 +10000 10000 -10000 +1000 1000 -1000 +100 100 -100 +10 10 -10 +1 1 -1 1000000000000000000 1000000000000000000 -1000000000000000000 -100000000000000000 100000000000000000.0 -100000000000000000.0 -10000000000000000 10000000000000000.00 -10000000000000000.00 -1000000000000000 1000000000000000.000 -1000000000000000.000 -100000000000000 100000000000000.0000 -100000000000000.0000 -10000000000000 10000000000000.00000 -10000000000000.00000 -1000000000000 1000000000000.000000 -1000000000000.000000 -100000000000 100000000000.0000000 -100000000000.0000000 -10000000000 10000000000.00000000 -10000000000.00000000 -1000000000 1000000000.000000000 -1000000000.000000000 -1000000000 1000000000.000000000 -1000000000.000000000 -100000000 100000000.0000000000 -100000000.0000000000 -10000000 10000000.00000000000 -10000000.00000000000 -1000000 1000000.000000000000 -1000000.000000000000 -100000 100000.0000000000000 -100000.0000000000000 -10000 10000.00000000000000 -10000.00000000000000 -1000 1000.000000000000000 -1000.000000000000000 -100 100.0000000000000000 -100.0000000000000000 -10 10.00000000000000000 -10.00000000000000000 -1 1.000000000000000000 -1.000000000000000000 -0.0000 0.00 0.00000000 -1.0000 0.11 0.11000000 -2.0000 0.22 0.22000000 -3.0000 0.33 0.33000000 -4.0000 0.44 0.44000000 -5.0000 0.55 0.55000000 -6.0000 0.66 0.66000000 -7.0000 0.77 0.77000000 -8.0000 0.88 0.88000000 -9.0000 1.00 1.00000000 -0.0000 0.00000000 0.00 -1.0000 0.11110000 0.11 -2.0000 0.22220000 0.22 -3.0000 0.33330000 0.33 -4.0000 0.44440000 0.44 -5.0000 0.55550000 0.55 -6.0000 0.66660000 0.66 -7.0000 0.77770000 0.77 -8.0000 0.88880000 0.88 -9.0000 1.00000000 1.00 -0.00000000 0.0000 0.00 -1.00000000 0.1111 0.11 -2.00000000 0.2222 0.22 -3.00000000 0.3333 0.33 -4.00000000 0.4444 0.44 -5.00000000 0.5555 0.55 -6.00000000 0.6666 0.66 -7.00000000 0.7777 0.77 -8.00000000 0.8888 0.88 -9.00000000 1.0000 1.00 -0.0000 0.00 0.00000000 -1.0000 0.11 0.11000000 -2.0000 0.22 0.22000000 -3.0000 0.33 0.33000000 -4.0000 0.44 0.44000000 -5.0000 0.55 0.55000000 -6.0000 0.66 0.66000000 -7.0000 0.77 0.77000000 -8.0000 0.88 0.88000000 -9.0000 1.00 1.00000000 -0.0000 0.00000000 0.00 -1.0000 0.11110000 0.11 -2.0000 0.22220000 0.22 -3.0000 0.33330000 0.33 -4.0000 0.44440000 0.44 -5.0000 0.55550000 0.55 -6.0000 0.66660000 0.66 -7.0000 0.77770000 0.77 -8.0000 0.88880000 0.88 -9.0000 1.00000000 1.00 -0.00000000 0.0000 0.00 -1.00000000 0.1111 0.11 -2.00000000 0.2222 0.22 -3.00000000 0.3333 0.33 -4.00000000 0.4444 0.44 -5.00000000 0.5555 0.55 -6.00000000 0.6666 0.66 -7.00000000 0.7777 0.77 -8.00000000 0.8888 0.88 -9.00000000 1.0000 1.00 +100000000000000000 100000000000000000 -100000000000000000 +10000000000000000 10000000000000000 -10000000000000000 +1000000000000000 1000000000000000 -1000000000000000 +100000000000000 100000000000000 -100000000000000 +10000000000000 10000000000000 -10000000000000 +1000000000000 1000000000000 -1000000000000 +100000000000 100000000000 -100000000000 +10000000000 10000000000 -10000000000 +1000000000 1000000000 -1000000000 +1000000000 1000000000 -1000000000 +100000000 100000000 -100000000 +10000000 10000000 -10000000 +1000000 1000000 -1000000 +100000 100000 -100000 +10000 10000 -10000 +1000 1000 -1000 +100 100 -100 +10 10 -10 +1 1 -1 +0 0 0 +1 0.11 0.11 +2 0.22 0.22 +3 0.33 0.33 +4 0.44 0.44 +5 0.55 0.55 +6 0.66 0.66 +7 0.77 0.77 +8 0.88 0.88 +9 1 1 +0 0 0 +1 0.1111 0.11 +2 0.2222 0.22 +3 0.3333 0.33 +4 0.4444 0.44 +5 0.5555 0.55 +6 0.6666 0.66 +7 0.7777 0.77 +8 0.8888 0.88 +9 1 1 +0 0 0 +1 0.1111 0.11 +2 0.2222 0.22 +3 0.3333 0.33 +4 0.4444 0.44 +5 0.5555 0.55 +6 0.6666 0.66 +7 0.7777 0.77 +8 0.8888 0.88 +9 1 1 +0 0 0 +1 0.11 0.11 +2 0.22 0.22 +3 0.33 0.33 +4 0.44 0.44 +5 0.55 0.55 +6 0.66 0.66 +7 0.77 0.77 +8 0.88 0.88 +9 1 1 +0 0 0 +1 0.1111 0.11 +2 0.2222 0.22 +3 0.3333 0.33 +4 0.4444 0.44 +5 0.5555 0.55 +6 0.6666 0.66 +7 0.7777 0.77 +8 0.8888 0.88 +9 1 1 +0 0 0 +1 0.1111 0.11 +2 0.2222 0.22 +3 0.3333 0.33 +4 0.4444 0.44 +5 0.5555 0.55 +6 0.6666 0.66 +7 0.7777 0.77 +8 0.8888 0.88 +9 1 1 99 99 -99 99 -99 9999 9999 -9999 9999 -9999 999999999 999999999 -999999999 999999999 -999999999 999999999 999999999 -999999999 999999999 -999999999 -999999999 999999999.000000000 -999999999.000000000 999999999.00000000000000000000000000000 -999999999.00000000000000000000000000000 -999999999 999999999.000000000 -999999999.000000000 999999999.00000000000000000000000000000 -999999999.00000000000000000000000000000 +999999999 999999999 -999999999 999999999 -999999999 +999999999 999999999 -999999999 999999999 -999999999 +999999999999999999 999999999999999999 -999999999999999999 999999999999999999 999999999999999999 -999999999999999999 999999999999999999 999999999999999999 -999999999999999999 -999999999999999999 999999999999999999.00000000000000000000 -999999999999999999.00000000000000000000 99 99 99 9999 9999 9999 999999999 999999999 999999999 999999999 999999999 999999999 42.42 42.42 42.42 42.42 -42.42 42.4200000 42.4200000000000000 42.420000000000000000000000000000000000 +42.42 42.42 42.42 42.42 123456789 123456789123456789 12345678901234567890123456789012345678 0.123456789 0.123456789123456789 diff --git a/tests/queries/0_stateless/00700_decimal_casts_2.reference b/tests/queries/0_stateless/00700_decimal_casts_2.reference index 393baae6c47..ed951e82036 100644 --- a/tests/queries/0_stateless/00700_decimal_casts_2.reference +++ b/tests/queries/0_stateless/00700_decimal_casts_2.reference @@ -1,36 +1,36 @@ -1234567890.0000000000000000000000000000 1234567890.00000000000000000000000000000 1234567890.00000000000000000000000000000 +1234567890 1234567890 1234567890 -126561577.683753853853498429727072845824 -1234567890.00000000 1234567890.000000000 1234567890.000000000 -12345678.0 12345678.00 12345678.00 -9223372036854775807.000000 9223372036854775807 -9223372036854775807 +1234567890 1234567890 1234567890 +12345678 12345678 12345678 +9223372036854775807 9223372036854775807 -9223372036854775807 9223372036854775800 9223372036854775800 -9223372036854775800 -92233720368547758.00 92233720368547758 -92233720368547758 -2147483647.0000000000 2147483647 -2147483647 -2147483647.00 2147483647 -2147483647 +92233720368547758 92233720368547758 -92233720368547758 +2147483647 2147483647 -2147483647 +2147483647 2147483647 -2147483647 92233720368547757.99 92233720368547757 -92233720368547757 2147483640.99 2147483640 -2147483640 --0.90000000 0 --0.90000000 0 --0.90000000 0 --0.8000 0 --0.8000 0 --0.8000 0 --0.70 0 --0.70 0 --0.70 0 --0.600000 0 --0.600000 0 --0.600000 0 +-0.9 0 +-0.9 0 +-0.9 0 +-0.8 0 +-0.8 0 +-0.8 0 +-0.7 0 +-0.7 0 +-0.7 0 +-0.6 0 +-0.6 0 +-0.6 0 +18446744073709551615 18446744073709551615 18446744073709551615 18446744073709551615 -18446744073709551615.00000000 18446744073709551615 4294967295 4294967295 -4294967295.0000000000 4294967295 4294967295 4294967295 -4294967295.0000 4294967295 +4294967295 4294967295 +4294967295 4294967295 +65535 65535 +65535 65535 65535 65535 -65535.0000000000 65535 65535 65535 -65535.0000 65535 2147483647 2147483647 -2147483647 -2147483647 2147483647 2147483647 diff --git a/tests/queries/0_stateless/00700_decimal_compare.reference b/tests/queries/0_stateless/00700_decimal_compare.reference index 2325847045f..6b2787642b7 100644 --- a/tests/queries/0_stateless/00700_decimal_compare.reference +++ b/tests/queries/0_stateless/00700_decimal_compare.reference @@ -2,27 +2,27 @@ 1 -42 -42 1 0 0 0 1 1 42 42 1 0 0 0 1 1 --42 -42.42000 0 0 1 1 0 1 -42 42.42000 0 1 0 1 1 0 +-42 -42.42 0 0 1 1 0 1 +42 42.42 0 1 0 1 1 0 1 1 1 0 0 0 -42 0 0 0 0 42 1 1 1 1 -42 0 0 0 0 42 1 1 1 1 -0.420000000 0.420000000000000000 0.42000000000000000000000000000000000000 -42.42 42.420000000 42.420000000000000000 42.42 --42.42 -42.420000000 -42.420000000000000000 -42.42 +0.42 0.42 0.42 +42.42 42.42 42.42 42.42 +-42.42 -42.42 -42.42 -42.42 42 42 42 42 42 42 42 42 42 42 42 42 --42 -42.42000 -42 -42.00000 -42 42.00000 42 42.42000 --42 -42 -42.42000 -0 0 0.00000 -0 0 0.00000 -42 42 42.42000 +-42 -42.42 -42 -42 +42 42 42 42.42 +-42 -42 -42.42 +0 0 0 +0 0 0 +42 42 42.42 1 0 1 0 1 0 @@ -35,5 +35,5 @@ 0 1 0 1 0 1 --42 -42 -42 -0.420000000 -0.420000000000000000 -0.42000000000000000000000000000000000000 -42.42000 -42.420000000 -42.420000000000000000 -42.42 -42 42 42 0.420000000 0.420000000000000000 0.42000000000000000000000000000000000000 42.42000 42.420000000 42.420000000000000000 42.42 +-42 -42 -42 -0.42 -0.42 -0.42 -42.42 -42.42 -42.42 -42.42 +42 42 42 0.42 0.42 0.42 42.42 42.42 42.42 42.42 diff --git a/tests/queries/0_stateless/00700_decimal_complex_types.reference b/tests/queries/0_stateless/00700_decimal_complex_types.reference index 9c7c6fefefd..b5ae11ad5d3 100644 --- a/tests/queries/0_stateless/00700_decimal_complex_types.reference +++ b/tests/queries/0_stateless/00700_decimal_complex_types.reference @@ -3,31 +3,31 @@ Array(Decimal(9, 2)) Array(Decimal(18, 2)) Array(Decimal(38, 2)) Decimal(9, 3) Decimal(18, 3) Decimal(38, 3) Decimal(9, 2) Decimal(18, 2) Decimal(38, 2) Tuple(Decimal(9, 1), Decimal(18, 1), Decimal(38, 1)) Decimal(9, 1) Decimal(18, 1) Decimal(38, 1) -0.100 -0.200 -0.300 -0.400 -0.500 -0.600 -0.700 -0.800 -0.900 +0.1 +0.2 +0.3 +0.4 +0.5 +0.6 +0.7 +0.8 +0.9 (9.1,9.2,9.3) 9.1 9.2 9.3 -[0.100,0.200,0.300] [0.100,0.200] [0.200,0.300] [0.100] [0.200] -[0.400,0.500,0.600] [0.400,0.500] [0.500,0.600] [0.400] [0.500] -[0.700,0.800,0.900] [0.700,0.800] [0.800,0.900] [0.700] [0.800] -[1.10,1.20] [1.10] [1.20] [1.10] [1.20] -[2.10,2.20] [2.10] [2.20] [2.10] [2.20] -[3.10,3.20] [3.10] [3.20] [3.10] [3.20] -[0.100,0.200,0.300,0.000] [0.000,0.100,0.200,0.300] -[0.400,0.500,0.600,0.000] [0.000,0.400,0.500,0.600] -[0.700,0.800,0.900,0.000] [0.000,0.700,0.800,0.900] -[0.100,0.200,0.300,0.000] Array(Decimal(9, 3)) -[0.400,0.500,0.600,0.000] Array(Decimal(18, 3)) -[0.700,0.800,0.900,0.000] Array(Decimal(38, 3)) -[0.0000,0.1000,0.2000,0.3000] Array(Decimal(9, 4)) -[0.0000,0.4000,0.5000,0.6000] Array(Decimal(18, 4)) -[0.0000,0.7000,0.8000,0.9000] Array(Decimal(38, 4)) +[0.1,0.2,0.3] [0.1,0.2] [0.2,0.3] [0.1] [0.2] +[0.4,0.5,0.6] [0.4,0.5] [0.5,0.6] [0.4] [0.5] +[0.7,0.8,0.9] [0.7,0.8] [0.8,0.9] [0.7] [0.8] +[1.1,1.2] [1.1] [1.2] [1.1] [1.2] +[2.1,2.2] [2.1] [2.2] [2.1] [2.2] +[3.1,3.2] [3.1] [3.2] [3.1] [3.2] +[0.1,0.2,0.3,0] [0,0.1,0.2,0.3] +[0.4,0.5,0.6,0] [0,0.4,0.5,0.6] +[0.7,0.8,0.9,0] [0,0.7,0.8,0.9] +[0.1,0.2,0.3,0] Array(Decimal(9, 3)) +[0.4,0.5,0.6,0] Array(Decimal(18, 3)) +[0.7,0.8,0.9,0] Array(Decimal(38, 3)) +[0,0.1,0.2,0.3] Array(Decimal(9, 4)) +[0,0.4,0.5,0.6] Array(Decimal(18, 4)) +[0,0.7,0.8,0.9] Array(Decimal(38, 4)) 3 3 3 2 2 2 0 0 0 @@ -66,24 +66,24 @@ Tuple(Decimal(9, 1), Decimal(18, 1), Decimal(38, 1)) Decimal(9, 1) Decimal(18, 1 1 1 1 -[0.100,0.200,0.300,0.400,0.500,0.600] Array(Decimal(18, 3)) -[0.100,0.200,0.300,0.700,0.800,0.900] Array(Decimal(38, 3)) -[0.400,0.500,0.600,0.700,0.800,0.900] Array(Decimal(38, 3)) -[0.100,0.200,0.300,1.100,1.200] Array(Decimal(9, 3)) -[0.400,0.500,0.600,2.100,2.200] Array(Decimal(18, 3)) -[0.700,0.800,0.900,3.100,3.200] Array(Decimal(38, 3)) -[0.100,0.200,0.300,2.100,2.200] Array(Decimal(18, 3)) -[0.100,0.200,0.300,3.100,3.200] Array(Decimal(38, 3)) -[0.400,0.500,0.600,1.100,1.200] Array(Decimal(18, 3)) -[0.400,0.500,0.600,3.100,3.200] Array(Decimal(38, 3)) -[0.700,0.800,0.900,1.100,1.200] Array(Decimal(38, 3)) -[0.700,0.800,0.900,2.100,2.200] Array(Decimal(38, 3)) +[0.1,0.2,0.3,0.4,0.5,0.6] Array(Decimal(18, 3)) +[0.1,0.2,0.3,0.7,0.8,0.9] Array(Decimal(38, 3)) +[0.4,0.5,0.6,0.7,0.8,0.9] Array(Decimal(38, 3)) +[0.1,0.2,0.3,1.1,1.2] Array(Decimal(9, 3)) +[0.4,0.5,0.6,2.1,2.2] Array(Decimal(18, 3)) +[0.7,0.8,0.9,3.1,3.2] Array(Decimal(38, 3)) +[0.1,0.2,0.3,2.1,2.2] Array(Decimal(18, 3)) +[0.1,0.2,0.3,3.1,3.2] Array(Decimal(38, 3)) +[0.4,0.5,0.6,1.1,1.2] Array(Decimal(18, 3)) +[0.4,0.5,0.6,3.1,3.2] Array(Decimal(38, 3)) +[0.7,0.8,0.9,1.1,1.2] Array(Decimal(38, 3)) +[0.7,0.8,0.9,2.1,2.2] Array(Decimal(38, 3)) 12345.6789 2 2 2 -12345.6789 2 2 2 123456789.123456784 2 2 2 -123456789.123456784 2 2 2 0.123456789123456784 2 2 2 --0.123456789112345680 2 2 2 +-0.12345678911234568 2 2 2 Decimal(9, 5) Decimal(9, 5) Decimal(9, 4) @@ -114,21 +114,21 @@ Decimal(38, 4) Decimal(38, 4) Decimal(9, 0) Decimal(18, 0) -32.20000 -32.10000 -64.20000 -32.10000 -128.20000 -32.10000 -32.20000 -64.10000 -64.20000 -64.10000 -128.20000 -64.10000 -32.20000 -128.10000 -64.20000 -128.10000 -128.20000 -128.10000 +32.2 +32.1 +64.2 +32.1 +128.2 +32.1 +32.2 +64.1 +64.2 +64.1 +128.2 +64.1 +32.2 +128.1 +64.2 +128.1 +128.2 +128.1 diff --git a/tests/queries/0_stateless/00700_decimal_defaults.reference b/tests/queries/0_stateless/00700_decimal_defaults.reference index f3f1fba83e7..04de9ac4c3d 100644 --- a/tests/queries/0_stateless/00700_decimal_defaults.reference +++ b/tests/queries/0_stateless/00700_decimal_defaults.reference @@ -4,11 +4,11 @@ c Decimal(38, 4) DEFAULT b / 3 d Decimal(9, 4) MATERIALIZED a + toDecimal32(\'0.2\', 1) e Decimal(18, 4) ALIAS b * 2 f Decimal(38, 4) ALIAS c * 6 -0.0000 0.0000 0.0000 -1.0000 0.5000 0.1666 -2.0000 1.0000 0.3333 -3.0000 1.5000 0.5000 -0.0000 0.0000 0.0000 0.2000 0.0000 0.0000 -1.0000 0.5000 0.1666 1.2000 1.0000 0.9996 -2.0000 1.0000 0.3333 2.2000 2.0000 1.9998 -3.0000 1.5000 0.5000 3.2000 3.0000 3.0000 +0 0 0 +1 0.5 0.1666 +2 1 0.3333 +3 1.5 0.5 +0 0 0 0.2 0 0 +1 0.5 0.1666 1.2 1 0.9996 +2 1 0.3333 2.2 2 1.9998 +3 1.5 0.5 3.2 3 3 diff --git a/tests/queries/0_stateless/00700_decimal_empty_aggregates.reference b/tests/queries/0_stateless/00700_decimal_empty_aggregates.reference index b079e91fddc..2c29b72f50c 100644 --- a/tests/queries/0_stateless/00700_decimal_empty_aggregates.reference +++ b/tests/queries/0_stateless/00700_decimal_empty_aggregates.reference @@ -1,50 +1,50 @@ 0 0 0 -[0.0000,0.0000] [0.0000000,0.0000000] [0.00000000,0.00000000] -0.0000 0.0000000 0.00000000 0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 0.0000 0.0000000 0.00000000 +[0,0] [0,0] [0,0] +0 0 0 0 0 0 +0 0 0 0 0 0 +0 0 0 0 0 0 +0 0 0 0 0 0 +0 0 0 0 0 0 (0,0,0) (0,0,0) (0,0,0) (0,0,0) (0,0,0) 0 0 0 0 0 0 -0.0000 0.0000 0.0000000 0.0000000 0.00000000 0.00000000 -0.0000 0.0000 0.0000000 0.0000000 0.00000000 0.00000000 -0.0000 0.0000 0.0000000 0.0000000 0.00000000 0.00000000 -0.0000 0.0000 0.0000000 0.0000000 0.00000000 0.00000000 -0.0000 0.0000000 0.00000000 Decimal(6, 4) Decimal(16, 7) Decimal(20, 8) -0.0000 0.0000000 0.00000000 Decimal(6, 4) Decimal(16, 7) Decimal(20, 8) -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -[0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000] -[0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000] -[0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000] -0.0000 0.0000000 0.00000000 Decimal(20, 8) -0.0000 0.0000000 0.00000000 Decimal(20, 8) -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -[0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000] -[0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000] -[0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000] -0.0000 0.0000000 0.00000000 Decimal(20, 8) -0.0000 0.0000000 0.00000000 Decimal(20, 8) -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -0.0000 0.0000000 0.00000000 -[0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000,0.0000] -[0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000,0.0000000] -[0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000,0.00000000] +0 0 0 0 0 0 +0 0 0 0 0 0 +0 0 0 0 0 0 +0 0 0 0 0 0 +0 0 0 Decimal(6, 4) Decimal(16, 7) Decimal(20, 8) +0 0 0 Decimal(6, 4) Decimal(16, 7) Decimal(20, 8) +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +[0,0,0,0,0,0,0,0,0,0,0] +[0,0,0,0,0,0,0,0,0,0,0] +[0,0,0,0,0,0,0,0,0,0,0] +0 0 0 Decimal(20, 8) +0 0 0 Decimal(20, 8) +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +[0,0,0,0,0,0,0,0,0,0,0] +[0,0,0,0,0,0,0,0,0,0,0] +[0,0,0,0,0,0,0,0,0,0,0] +0 0 0 Decimal(20, 8) +0 0 0 Decimal(20, 8) +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +0 0 0 +[0,0,0,0,0,0,0,0,0,0,0] +[0,0,0,0,0,0,0,0,0,0,0] +[0,0,0,0,0,0,0,0,0,0,0] inf inf inf Float64 Float64 Float64 nan nan nan nan nan nan Float64 Float64 Float64 diff --git a/tests/queries/0_stateless/00700_decimal_formats.reference b/tests/queries/0_stateless/00700_decimal_formats.reference index 0bea4ba27be..ff3961780a2 100644 --- a/tests/queries/0_stateless/00700_decimal_formats.reference +++ b/tests/queries/0_stateless/00700_decimal_formats.reference @@ -1,42 +1,42 @@ {"a":0.055,"b":-0.000000005,"c":0.000000000000000005} -{"a":0.100,"b":-0.100000000,"c":0.100000000000000000} -{"a":0.200,"b":-0.200000000,"c":0.200000000000000000} -{"a":0.300,"b":-0.300000000,"c":0.300000000000000000} -{"a":0.420,"b":-0.420000000,"c":0.420000000000000000} -{"a":1.000,"b":-1.000000000,"c":1.000000000000000000} -{"a":1.100,"b":-1.100000000,"c":1.100000000000000000} -{"a":2.000,"b":-2.000000000,"c":2.000000000000000000} -{"a":2.200,"b":-2.200000000,"c":2.200000000000000000} -{"a":3.000,"b":-3.000000000,"c":3.000000000000000000} -{"a":3.300,"b":-3.300000000,"c":3.300000000000000000} -{"a":42.000,"b":-42.000000000,"c":42.000000000000000000} -{"a":42.420,"b":-42.420000000,"c":42.420000000000000000} -{"a":440000.000,"b":-400000000.000000000,"c":40000000000000000000.000000000000000000} +{"a":0.1,"b":-0.1,"c":0.1} +{"a":0.2,"b":-0.2,"c":0.2} +{"a":0.3,"b":-0.3,"c":0.3} +{"a":0.42,"b":-0.42,"c":0.42} +{"a":1,"b":-1,"c":1} +{"a":1.1,"b":-1.1,"c":1.1} +{"a":2,"b":-2,"c":2} +{"a":2.2,"b":-2.2,"c":2.2} +{"a":3,"b":-3,"c":3} +{"a":3.3,"b":-3.3,"c":3.3} +{"a":42,"b":-42,"c":42} +{"a":42.42,"b":-42.42,"c":42.42} +{"a":440000,"b":-400000000,"c":40000000000000000000} 0.055,-0.000000005,0.000000000000000005 -0.100,-0.100000000,0.100000000000000000 -0.200,-0.200000000,0.200000000000000000 -0.300,-0.300000000,0.300000000000000000 -0.420,-0.420000000,0.420000000000000000 -1.000,-1.000000000,1.000000000000000000 -1.100,-1.100000000,1.100000000000000000 -2.000,-2.000000000,2.000000000000000000 -2.200,-2.200000000,2.200000000000000000 -3.000,-3.000000000,3.000000000000000000 -3.300,-3.300000000,3.300000000000000000 -42.000,-42.000000000,42.000000000000000000 -42.420,-42.420000000,42.420000000000000000 -440000.000,-400000000.000000000,40000000000000000000.000000000000000000 +0.1,-0.1,0.1 +0.2,-0.2,0.2 +0.3,-0.3,0.3 +0.42,-0.42,0.42 +1,-1,1 +1.1,-1.1,1.1 +2,-2,2 +2.2,-2.2,2.2 +3,-3,3 +3.3,-3.3,3.3 +42,-42,42 +42.42,-42.42,42.42 +440000,-400000000,40000000000000000000 0.055 -0.000000005 0.000000000000000005 -0.100 -0.100000000 0.100000000000000000 -0.200 -0.200000000 0.200000000000000000 -0.300 -0.300000000 0.300000000000000000 -0.420 -0.420000000 0.420000000000000000 -1.000 -1.000000000 1.000000000000000000 -1.100 -1.100000000 1.100000000000000000 -2.000 -2.000000000 2.000000000000000000 -2.200 -2.200000000 2.200000000000000000 -3.000 -3.000000000 3.000000000000000000 -3.300 -3.300000000 3.300000000000000000 -42.000 -42.000000000 42.000000000000000000 -42.420 -42.420000000 42.420000000000000000 -440000.000 -400000000.000000000 40000000000000000000.000000000000000000 +0.1 -0.1 0.1 +0.2 -0.2 0.2 +0.3 -0.3 0.3 +0.42 -0.42 0.42 +1 -1 1 +1.1 -1.1 1.1 +2 -2 2 +2.2 -2.2 2.2 +3 -3 3 +3.3 -3.3 3.3 +42 -42 42 +42.42 -42.42 42.42 +440000 -400000000 40000000000000000000 diff --git a/tests/queries/0_stateless/00700_decimal_gathers.reference b/tests/queries/0_stateless/00700_decimal_gathers.reference index bbfd7388e12..273642d15a8 100644 --- a/tests/queries/0_stateless/00700_decimal_gathers.reference +++ b/tests/queries/0_stateless/00700_decimal_gathers.reference @@ -1,13 +1,13 @@ -[2.000] -[2.0000000000] -[2.000000000000000000] -[1.000] -[1.0000000000] -[1.000000000000000000] -- -[2.000] -[1] -[2.000000000000000000] -[1.000] [2] -[1.000000000000000000] +[2] +[2] +[1] +[1] +[1] +- +[2] +[1] +[2] +[1] +[2] +[1] diff --git a/tests/queries/0_stateless/00700_decimal_in_keys.reference b/tests/queries/0_stateless/00700_decimal_in_keys.reference index ec11144a206..4b3486b5ce7 100644 --- a/tests/queries/0_stateless/00700_decimal_in_keys.reference +++ b/tests/queries/0_stateless/00700_decimal_in_keys.reference @@ -5,25 +5,25 @@ 1 1 5 -9.00000000 29.00000000 29.00000000 -8.00000000 28.00000000 28.00000000 -7.00000000 27.00000000 27.00000000 -6.00000000 26.00000000 26.00000000 -9.00000000 19.00000000 19.00000000 -8.00000000 18.00000000 18.00000000 -7.00000000 17.00000000 17.00000000 -6.00000000 16.00000000 16.00000000 -9.00000000 9.00000000 9.00000000 -8.00000000 8.00000000 8.00000000 -7.00000000 7.00000000 7.00000000 -6.00000000 6.00000000 6.00000000 -1.00000000 1.00000000 1.00000000 -3.00000000 3.00000000 3.00000000 -1.00000000 11.00000000 11.00000000 -3.00000000 13.00000000 13.00000000 -1.00000000 21.00000000 21.00000000 -3.00000000 23.00000000 23.00000000 -1.00000000 31.00000000 31.00000000 -3.00000000 33.00000000 33.00000000 -1.00000000 41.00000000 41.00000000 -3.00000000 43.00000000 43.00000000 +9 29 29 +8 28 28 +7 27 27 +6 26 26 +9 19 19 +8 18 18 +7 17 17 +6 16 16 +9 9 9 +8 8 8 +7 7 7 +6 6 6 +1 1 1 +3 3 3 +1 11 11 +3 13 13 +1 21 21 +3 23 23 +1 31 31 +3 33 33 +1 41 41 +3 43 43 diff --git a/tests/queries/0_stateless/00700_decimal_math.reference b/tests/queries/0_stateless/00700_decimal_math.reference index f58e08dc1fb..eb556ac49b8 100644 --- a/tests/queries/0_stateless/00700_decimal_math.reference +++ b/tests/queries/0_stateless/00700_decimal_math.reference @@ -1,30 +1,30 @@ -42.4200 3.7476 42.419154 -42.4200 5.4066 42.417862 -42.4200 1.6275 42.413098 -42.4200 6.513 42.419169 -42.4200 3.4875 42.417263671875 -1.00000 0.8427007929497149 0.15729920705028513 -42.4200 115.60113124678627 1.6029995567009473e50 -0.00 0 1 0 +42.42 3.7476 42.419154 +42.42 5.4066 42.417862 +42.42 1.6275 42.413098 +42.42 6.513 42.419169 +42.42 3.4875 42.417263671875 +1 0.8427007929497149 0.15729920705028513 +42.42 115.60113124678627 1.6029995567009473e50 +0 0 1 0 3.14159265 0 -1 -0 -1.00 1.5707963267948966 0 0.7853981633974483 -42.4200 3.7476 42.419154 -42.4200 5.4066 42.417862 -42.4200 1.6275 42.413098 -42.4200 6.513 42.419169 -42.4200 3.4875 42.417263671875 -1.00000 0.8427007929497149 0.15729920705028513 -42.4200 115.60113124678627 1.6029995567009473e50 -0.00 0 1 0 +1 1.5707963267948966 0 0.7853981633974483 +42.42 3.7476 42.419154 +42.42 5.4066 42.417862 +42.42 1.6275 42.413098 +42.42 6.513 42.419169 +42.42 3.4875 42.417263671875 +1 0.8427007929497149 0.15729920705028513 +42.42 115.60113124678627 1.6029995567009473e50 +0 0 1 0 3.14159265358979328 0 -1 -0 -1.00 1.5707963267948966 0 0.7853981633974483 -42.4200 3.7476 42.419154 -42.4200 5.4066 42.417862 -42.4200 1.6275 42.413098 -42.4200 6.513 42.419169 -42.4200 3.4875 42.417263671875 -1.00000 0.8427007929497149 0.15729920705028513 -42.4200 115.60113124678627 1.6029995567009473e50 -0.00 0 1 0 +1 1.5707963267948966 0 0.7853981633974483 +42.42 3.7476 42.419154 +42.42 5.4066 42.417862 +42.42 1.6275 42.413098 +42.42 6.513 42.419169 +42.42 3.4875 42.417263671875 +1 0.8427007929497149 0.15729920705028513 +42.42 115.60113124678627 1.6029995567009473e50 +0 0 1 0 3.14159265358979 0 -1 -0 -1.00 1.5707963267948966 0 0.7853981633974483 +1 1.5707963267948966 0 0.7853981633974483 diff --git a/tests/queries/0_stateless/00700_decimal_null.reference b/tests/queries/0_stateless/00700_decimal_null.reference index 250a437a883..e9ddf011260 100644 --- a/tests/queries/0_stateless/00700_decimal_null.reference +++ b/tests/queries/0_stateless/00700_decimal_null.reference @@ -17,11 +17,11 @@ \N 1 1 -1.10 1.10000 1.10000 1.1000 1.10000000 1.10000000 -2.20 2.20000 2.20000 2.2000 \N \N -3.30 3.30000 3.30000 \N 3.30000000 \N -4.40 4.40000 4.40000 \N \N 4.40000000 -5.50 5.50000 5.50000 \N \N \N +1.1 1.1 1.1 1.1 1.1 1.1 +2.2 2.2 2.2 2.2 \N \N +3.3 3.3 3.3 \N 3.3 \N +4.4 4.4 4.4 \N \N 4.4 +5.5 5.5 5.5 \N \N \N 0 1 0 1 0 1 diff --git a/tests/queries/0_stateless/00700_decimal_round.reference b/tests/queries/0_stateless/00700_decimal_round.reference index 230b6863411..d0f03c07849 100644 --- a/tests/queries/0_stateless/00700_decimal_round.reference +++ b/tests/queries/0_stateless/00700_decimal_round.reference @@ -1,75 +1,75 @@ -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12345.0000 12345.6000 12345.6700 12345.6780 12345.6789 12345.6789 -12345.6789 12345.0000 12345.6000 12345.6700 12345.6780 12345.6789 12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12345.0000 -12345.6000 -12345.6700 -12345.6780 -12345.6789 -12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12345.0000 -12345.6000 -12345.6700 -12345.6780 -12345.6789 -12345.6789 -12345.6789 12350.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12350.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12350.0000 12400.0000 13000.0000 20000.0000 100000.0000 -12345.6789 12340.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12340.0000 12300.0000 12000.0000 10000.0000 0.0000 --12345.6789 -12350.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12350.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12340.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12350.0000 -12400.0000 -13000.0000 -20000.0000 -100000.0000 --12345.6789 -12340.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12345.0000 12345.6000 12345.6700 12345.6780 12345.6789 12345.6789 -12345.6789 12345.0000 12345.6000 12345.6700 12345.6780 12345.6789 12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12345.0000 -12345.6000 -12345.6700 -12345.6780 -12345.6789 -12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12345.0000 -12345.6000 -12345.6700 -12345.6780 -12345.6789 -12345.6789 -12345.6789 12350.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12350.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12350.0000 12400.0000 13000.0000 20000.0000 100000.0000 -12345.6789 12340.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12340.0000 12300.0000 12000.0000 10000.0000 0.0000 --12345.6789 -12350.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12350.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12340.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12350.0000 -12400.0000 -13000.0000 -20000.0000 -100000.0000 --12345.6789 -12340.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12346.0000 12345.7000 12345.6800 12345.6790 12345.6789 12345.6789 -12345.6789 12345.0000 12345.6000 12345.6700 12345.6780 12345.6789 12345.6789 -12345.6789 12345.0000 12345.6000 12345.6700 12345.6780 12345.6789 12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12345.0000 -12345.6000 -12345.6700 -12345.6780 -12345.6789 -12345.6789 --12345.6789 -12346.0000 -12345.7000 -12345.6800 -12345.6790 -12345.6789 -12345.6789 --12345.6789 -12345.0000 -12345.6000 -12345.6700 -12345.6780 -12345.6789 -12345.6789 -12345.6789 12350.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12350.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12350.0000 12400.0000 13000.0000 20000.0000 100000.0000 -12345.6789 12340.0000 12300.0000 12000.0000 10000.0000 0.0000 -12345.6789 12340.0000 12300.0000 12000.0000 10000.0000 0.0000 --12345.6789 -12350.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12350.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12340.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 --12345.6789 -12350.0000 -12400.0000 -13000.0000 -20000.0000 -100000.0000 --12345.6789 -12340.0000 -12300.0000 -12000.0000 -10000.0000 0.0000 -123456789.123456789 -123456789.123456789 123456789.000000000 -123456789.000000000 123456789.123460000 -123456789.123460000 123500000.000000000 -123500000.000000000 -123456789.123456789 -123456789.123456789 123456789.000000000 -123456789.000000000 123456789.123460000 -123456789.123460000 123500000.000000000 -123500000.000000000 -123456789.123456789 -123456789.123456789 123456790.000000000 -123456789.000000000 123456789.123460000 -123456789.123450000 123500000.000000000 -123400000.000000000 -123456789.123456789 -123456789.123456789 123456789.000000000 -123456790.000000000 123456789.123450000 -123456789.123460000 123400000.000000000 -123500000.000000000 -123456789.123456789 -123456789.123456789 123456789.000000000 -123456789.000000000 123456789.123450000 -123456789.123450000 123400000.000000000 -123400000.000000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789.000000000 -12345678901234567890123456789.000000000 12345678901234567890123456789.123000000 -12345678901234567890123456789.123000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789.000000000 -12345678901234567890123456789.000000000 12345678901234567890123456789.123000000 -12345678901234567890123456789.123000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456790.000000000 -12345678901234567890123456789.000000000 12345678901234567890123456789.124000000 -12345678901234567890123456789.123000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789.000000000 -12345678901234567890123456790.000000000 12345678901234567890123456789.123000000 -12345678901234567890123456789.124000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789.000000000 -12345678901234567890123456789.000000000 12345678901234567890123456789.123000000 -12345678901234567890123456789.123000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789.000000000 -12345678901234567890123456789.000000000 12345678901234567890123457000.000000000 -12345678901234567890123457000.000000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789.000000000 -12345678901234567890123456789.000000000 12345678901234567890123457000.000000000 -12345678901234567890123457000.000000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456790.000000000 -12345678901234567890123456789.000000000 12345678901234567890123457000.000000000 -12345678901234567890123456000.000000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789.000000000 -12345678901234567890123456790.000000000 12345678901234567890123456000.000000000 -12345678901234567890123457000.000000000 -12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789.000000000 -12345678901234567890123456789.000000000 12345678901234567890123456000.000000000 -12345678901234567890123456000.000000000 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12345 12345.6 12345.67 12345.678 12345.6789 12345.6789 +12345.6789 12345 12345.6 12345.67 12345.678 12345.6789 12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12345 -12345.6 -12345.67 -12345.678 -12345.6789 -12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12345 -12345.6 -12345.67 -12345.678 -12345.6789 -12345.6789 +12345.6789 12350 12300 12000 10000 0 +12345.6789 12350 12300 12000 10000 0 +12345.6789 12350 12400 13000 20000 100000 +12345.6789 12340 12300 12000 10000 0 +12345.6789 12340 12300 12000 10000 0 +-12345.6789 -12350 -12300 -12000 -10000 0 +-12345.6789 -12350 -12300 -12000 -10000 0 +-12345.6789 -12340 -12300 -12000 -10000 0 +-12345.6789 -12350 -12400 -13000 -20000 -100000 +-12345.6789 -12340 -12300 -12000 -10000 0 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12345 12345.6 12345.67 12345.678 12345.6789 12345.6789 +12345.6789 12345 12345.6 12345.67 12345.678 12345.6789 12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12345 -12345.6 -12345.67 -12345.678 -12345.6789 -12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12345 -12345.6 -12345.67 -12345.678 -12345.6789 -12345.6789 +12345.6789 12350 12300 12000 10000 0 +12345.6789 12350 12300 12000 10000 0 +12345.6789 12350 12400 13000 20000 100000 +12345.6789 12340 12300 12000 10000 0 +12345.6789 12340 12300 12000 10000 0 +-12345.6789 -12350 -12300 -12000 -10000 0 +-12345.6789 -12350 -12300 -12000 -10000 0 +-12345.6789 -12340 -12300 -12000 -10000 0 +-12345.6789 -12350 -12400 -13000 -20000 -100000 +-12345.6789 -12340 -12300 -12000 -10000 0 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12346 12345.7 12345.68 12345.679 12345.6789 12345.6789 +12345.6789 12345 12345.6 12345.67 12345.678 12345.6789 12345.6789 +12345.6789 12345 12345.6 12345.67 12345.678 12345.6789 12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12345 -12345.6 -12345.67 -12345.678 -12345.6789 -12345.6789 +-12345.6789 -12346 -12345.7 -12345.68 -12345.679 -12345.6789 -12345.6789 +-12345.6789 -12345 -12345.6 -12345.67 -12345.678 -12345.6789 -12345.6789 +12345.6789 12350 12300 12000 10000 0 +12345.6789 12350 12300 12000 10000 0 +12345.6789 12350 12400 13000 20000 100000 +12345.6789 12340 12300 12000 10000 0 +12345.6789 12340 12300 12000 10000 0 +-12345.6789 -12350 -12300 -12000 -10000 0 +-12345.6789 -12350 -12300 -12000 -10000 0 +-12345.6789 -12340 -12300 -12000 -10000 0 +-12345.6789 -12350 -12400 -13000 -20000 -100000 +-12345.6789 -12340 -12300 -12000 -10000 0 +123456789.123456789 -123456789.123456789 123456789 -123456789 123456789.12346 -123456789.12346 123500000 -123500000 +123456789.123456789 -123456789.123456789 123456789 -123456789 123456789.12346 -123456789.12346 123500000 -123500000 +123456789.123456789 -123456789.123456789 123456790 -123456789 123456789.12346 -123456789.12345 123500000 -123400000 +123456789.123456789 -123456789.123456789 123456789 -123456790 123456789.12345 -123456789.12346 123400000 -123500000 +123456789.123456789 -123456789.123456789 123456789 -123456789 123456789.12345 -123456789.12345 123400000 -123400000 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789 -12345678901234567890123456789 12345678901234567890123456789.123 -12345678901234567890123456789.123 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789 -12345678901234567890123456789 12345678901234567890123456789.123 -12345678901234567890123456789.123 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456790 -12345678901234567890123456789 12345678901234567890123456789.124 -12345678901234567890123456789.123 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789 -12345678901234567890123456790 12345678901234567890123456789.123 -12345678901234567890123456789.124 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789 -12345678901234567890123456789 12345678901234567890123456789.123 -12345678901234567890123456789.123 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789 -12345678901234567890123456789 12345678901234567890123457000 -12345678901234567890123457000 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789 -12345678901234567890123456789 12345678901234567890123457000 -12345678901234567890123457000 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456790 -12345678901234567890123456789 12345678901234567890123457000 -12345678901234567890123456000 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789 -12345678901234567890123456790 12345678901234567890123456000 -12345678901234567890123457000 +12345678901234567890123456789.123456789 -12345678901234567890123456789.123456789 12345678901234567890123456789 -12345678901234567890123456789 12345678901234567890123456000 -12345678901234567890123456000 diff --git a/tests/queries/0_stateless/00700_to_decimal_or_something.reference b/tests/queries/0_stateless/00700_to_decimal_or_something.reference index 7a6ff87d096..89ded7bd6d4 100644 --- a/tests/queries/0_stateless/00700_to_decimal_or_something.reference +++ b/tests/queries/0_stateless/00700_to_decimal_or_something.reference @@ -1,8 +1,8 @@ -1.1 1.10 1.10000000 +1.1 1.1 1.1 0 0 0.42 -0 0.420 -0 0.4200 +0 0.42 +0 0.42 999999999 0 -999999999 0 999999999999999999 0 @@ -12,11 +12,11 @@ -99999999999999999999999999999999999999 0 ---- -1.1 1.10 1.10000000 +1.1 1.1 1.1 \N \N -0.42 -\N -0.420 -\N -0.4200 +\N -0.42 +\N -0.42 999999999 \N -999999999 \N 999999999999999999 \N diff --git a/tests/queries/0_stateless/00732_decimal_summing_merge_tree.reference b/tests/queries/0_stateless/00732_decimal_summing_merge_tree.reference index 1644f50993c..551ef8f59c5 100644 --- a/tests/queries/0_stateless/00732_decimal_summing_merge_tree.reference +++ b/tests/queries/0_stateless/00732_decimal_summing_merge_tree.reference @@ -1,2 +1,2 @@ -2001-01-01 2.0000 0.00000000 -2.0000000000 -2001-01-01 0.0000 1.00000000 0.0000000000 +2001-01-01 2 0 -2 +2001-01-01 0 1 0 diff --git a/tests/queries/0_stateless/00737_decimal_group_by.reference b/tests/queries/0_stateless/00737_decimal_group_by.reference index 2f838f4bcdd..3e7ca2bf83b 100644 --- a/tests/queries/0_stateless/00737_decimal_group_by.reference +++ b/tests/queries/0_stateless/00737_decimal_group_by.reference @@ -1,11 +1,11 @@ -1.10 -2.1000 -3.100000000000 -1.20 -2.2000 -3.200000000000 -1.30 -2.3000 -3.300000000000 -1 1.000000000000000000 10.000000000000000000 -1 1.000000000000000000 10.000000000000000000 +1.1 +2.1 +3.1 +1.2 +2.2 +3.2 +1.3 +2.3 +3.3 +1 1 10 +1 1 10 diff --git a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference index 6470739db21..7bd91e5a69b 100644 --- a/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference +++ b/tests/queries/0_stateless/00804_test_custom_compression_codecs.reference @@ -11,7 +11,7 @@ 9175437371954010821 CREATE TABLE default.compression_codec_multiple_more_types\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), NONE, NONE, NONE, LZ4HC(0)),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = MergeTree\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] -7.1000000000000 xxxxxxxxxxxx [127] ['Henry'] +7.1 xxxxxxxxxxxx [127] ['Henry'] ! 222 !ZSTD diff --git a/tests/queries/0_stateless/00805_round_down.reference b/tests/queries/0_stateless/00805_round_down.reference index 0ed04c7757f..dafbf4255fc 100644 --- a/tests/queries/0_stateless/00805_round_down.reference +++ b/tests/queries/0_stateless/00805_round_down.reference @@ -51,33 +51,33 @@ 7 6.25 8 7.5 9 8.75 -0.00000 4.00000 -0.01000 4.00000 -0.02000 4.00000 -0.03000 4.00000 -0.04000 4.00000 -0.05000 4.00000 -0.06000 4.00000 -0.07000 4.00000 -0.08000 4.00000 -0.09000 4.00000 -0.00000 0.04000 -0.01000 0.04000 -0.02000 0.04000 -0.03000 0.04000 -0.04000 0.04000 -0.05000 0.05000 -0.06000 0.06000 -0.07000 0.06000 -0.08000 0.06000 -0.09000 0.06000 -0.00000 0.04000 -0.01000 0.04000 -0.02000 0.04000 -0.03000 0.04000 -0.04000 0.04000 -0.05000 0.05000 -0.06000 0.06000 -0.07000 0.06000 -0.08000 0.06000 -0.09000 0.06000 +0 4 +0.01 4 +0.02 4 +0.03 4 +0.04 4 +0.05 4 +0.06 4 +0.07 4 +0.08 4 +0.09 4 +0 0.04 +0.01 0.04 +0.02 0.04 +0.03 0.04 +0.04 0.04 +0.05 0.05 +0.06 0.06 +0.07 0.06 +0.08 0.06 +0.09 0.06 +0 0.04 +0.01 0.04 +0.02 0.04 +0.03 0.04 +0.04 0.04 +0.05 0.05 +0.06 0.06 +0.07 0.06 +0.08 0.06 +0.09 0.06 diff --git a/tests/queries/0_stateless/00837_minmax_index.reference b/tests/queries/0_stateless/00837_minmax_index.reference index 0f5a8eb904e..8bde896b02a 100644 --- a/tests/queries/0_stateless/00837_minmax_index.reference +++ b/tests/queries/0_stateless/00837_minmax_index.reference @@ -1,6 +1,6 @@ -0 5 4.7 6.50 cba b 2014-01-04 -1 5 4.7 6.50 cba b 2014-03-11 -11 5 4.7 6.50 cba b 2014-06-11 -12 5 4.7 6.50 cba b 2015-01-01 +0 5 4.7 6.5 cba b 2014-01-04 +1 5 4.7 6.5 cba b 2014-03-11 +11 5 4.7 6.5 cba b 2014-06-11 +12 5 4.7 6.5 cba b 2015-01-01 "rows_read": 4, "rows_read": 2, diff --git a/tests/queries/0_stateless/00838_unique_index.reference b/tests/queries/0_stateless/00838_unique_index.reference index df890188102..7183a3b7370 100644 --- a/tests/queries/0_stateless/00838_unique_index.reference +++ b/tests/queries/0_stateless/00838_unique_index.reference @@ -1,24 +1,24 @@ -0 5 4.7 6.50 cba b 2014-01-04 -1 5 4.7 6.50 cba b 2014-03-11 -12 5 4.7 6.50 cba b 2014-06-11 -13 5 4.7 6.50 cba b 2015-01-01 +0 5 4.7 6.5 cba b 2014-01-04 +1 5 4.7 6.5 cba b 2014-03-11 +12 5 4.7 6.5 cba b 2014-06-11 +13 5 4.7 6.5 cba b 2015-01-01 "rows_read": 4, -2 2 4.5 2.50 abc a 2014-01-01 -6 2 4.5 2.50 abc a 2014-02-11 +2 2 4.5 2.5 abc a 2014-01-01 +6 2 4.5 2.5 abc a 2014-02-11 7 5 6.9 1.57 bac c 2014-04-11 -8 2 4.5 2.50 abc a 2014-05-11 +8 2 4.5 2.5 abc a 2014-05-11 9 5 6.9 1.57 bac c 2014-07-11 5 5 6.9 1.57 bac c 2014-11-11 -4 2 4.5 2.50 abc a 2016-01-01 +4 2 4.5 2.5 abc a 2016-01-01 3 5 6.9 1.57 bac c 2017-01-01 "rows_read": 8, "rows_read": 2, -2 2 4.5 2.50 abc a 2014-01-01 -6 2 4.5 2.50 abc a 2014-02-11 +2 2 4.5 2.5 abc a 2014-01-01 +6 2 4.5 2.5 abc a 2014-02-11 7 5 6.9 1.57 bac c 2014-04-11 -8 2 4.5 2.50 abc a 2014-05-11 +8 2 4.5 2.5 abc a 2014-05-11 9 5 6.9 1.57 bac c 2014-07-11 5 5 6.9 1.57 bac c 2014-11-11 -4 2 4.5 2.50 abc a 2016-01-01 +4 2 4.5 2.5 abc a 2016-01-01 3 5 6.9 1.57 bac c 2017-01-01 "rows_read": 8, diff --git a/tests/queries/0_stateless/00861_decimal_quoted_csv.reference b/tests/queries/0_stateless/00861_decimal_quoted_csv.reference index 6a219226835..3ff285acef8 100644 --- a/tests/queries/0_stateless/00861_decimal_quoted_csv.reference +++ b/tests/queries/0_stateless/00861_decimal_quoted_csv.reference @@ -1,5 +1,5 @@ -1 1.00 1.00 1.00 -2 -1.00 -1.00 -1.00 -3 1.00 1.00 1.00 -4 -0.10 -0.10 -0.10 +1 1 1 1 +2 -1 -1 -1 +3 1 1 1 +4 -0.1 -0.1 -0.1 5 0.01 0.01 0.01 diff --git a/tests/queries/0_stateless/00862_decimal_in.reference b/tests/queries/0_stateless/00862_decimal_in.reference index 2e4eb5e6dc7..0cd93f69c38 100644 --- a/tests/queries/0_stateless/00862_decimal_in.reference +++ b/tests/queries/0_stateless/00862_decimal_in.reference @@ -1,18 +1,18 @@ -128.00 128.00 -128.00 128.00 -128.00 128.00 -128.00 128.00 -128.00 128.00 -128.00 128.00 -32.00 32.00 -32.00 32.00 -32.00 32.00 -32.00 32.00 -32.00 32.00 -32.00 32.00 -64.00 64.00 -64.00 64.00 -64.00 64.00 -64.00 64.00 -64.00 64.00 -64.00 64.00 +128 128 +128 128 +128 128 +128 128 +128 128 +128 128 +32 32 +32 32 +32 32 +32 32 +32 32 +32 32 +64 64 +64 64 +64 64 +64 64 +64 64 +64 64 diff --git a/tests/queries/0_stateless/00910_decimal_group_array_crash_3783.reference b/tests/queries/0_stateless/00910_decimal_group_array_crash_3783.reference index 232d9aa7974..47e910f691d 100644 --- a/tests/queries/0_stateless/00910_decimal_group_array_crash_3783.reference +++ b/tests/queries/0_stateless/00910_decimal_group_array_crash_3783.reference @@ -1,9 +1,9 @@ -[1.00] -[1.00000] -[1.0000000000] -[499500.00] -[499500.00000] -[499500.0000000000] +[1] +[1] +[1] +[499500] +[499500] +[499500] 1545081300 [('ed87e57c-9331-462a-80b4-9f0c005e88c8',0.44)] -4341757 5657967 2018-11-01 16:47:46 txt 321.380000000000 315.080000000000 0.000000000000 2018-11-02 00:00:00 -4360430 5681495 2018-11-02 09:00:07 txt 274.350000000000 268.970000000000 0.000000000000 2018-11-02 00:00:00 +4341757 5657967 2018-11-01 16:47:46 txt 321.38 315.08 0 2018-11-02 00:00:00 +4360430 5681495 2018-11-02 09:00:07 txt 274.35 268.97 0 2018-11-02 00:00:00 diff --git a/tests/queries/0_stateless/00927_asof_join_other_types.reference b/tests/queries/0_stateless/00927_asof_join_other_types.reference index a34437f66c2..83ee534ff91 100644 --- a/tests/queries/0_stateless/00927_asof_join_other_types.reference +++ b/tests/queries/0_stateless/00927_asof_join_other_types.reference @@ -13,15 +13,15 @@ 2 1970-01-01 03:00:01 1 0 2 1970-01-01 03:00:03 3 3 2 1970-01-01 03:00:05 5 3 -2 1.00000 1 0 -2 3.00000 3 3 -2 5.00000 5 3 -2 1.00000 1 0 -2 3.00000 3 3 -2 5.00000 5 3 -2 1.00000 1 0 -2 3.00000 3 3 -2 5.00000 5 3 +2 1 1 0 +2 3 3 3 +2 5 5 3 +2 1 1 0 +2 3 3 3 +2 5 5 3 +2 1 1 0 +2 3 3 3 +2 5 5 3 2 1970-01-01 03:00:00.001 1 0 2 1970-01-01 03:00:00.003 3 3 2 1970-01-01 03:00:00.005 5 3 diff --git a/tests/queries/0_stateless/00950_dict_get.reference b/tests/queries/0_stateless/00950_dict_get.reference index c1b502bf773..191eb40a889 100644 --- a/tests/queries/0_stateless/00950_dict_get.reference +++ b/tests/queries/0_stateless/00950_dict_get.reference @@ -31,18 +31,18 @@ dictGetOrDefault complex_hashed_strings 0 * * dictGet complex_cache_strings 1 1 1 dictGetOrDefault complex_cache_strings 1 1 1 dictGetOrDefault complex_cache_strings 0 * * -dictGet flat_decimals 1 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault flat_decimals 1 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault flat_decimals 0 42.0000 42.000000 42.0 (42.0000,42.000000,42.0) -dictGet hashed_decimals 1 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault hashed_decimals 1 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault hashed_decimals 0 42.0000 42.000000 42.0 (42.0000,42.000000,42.0) -dictGet cache_decimals 1 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault cache_decimals 1 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault cache_decimals 0 42.0000 42.000000 42.0 (42.0000,42.000000,42.0) -dictGet complex_hashed_decimals (1) 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault complex_hashed_decimals (1) 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault complex_hashed_decimals (0) 42.0000 42.000000 42.0 (42.0000,42.000000,42.0) -dictGet complex_cache_decimals (1) 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault complex_cache_decimals (1) 1.0000 1.000000 1.0 (1.0000,1.000000,1.0) -dictGetOrDefault complex_cache_decimals (0) 42.0000 42.000000 42.0 (42.0000,42.000000,42.0) +dictGet flat_decimals 1 1 1 1 (1,1,1) +dictGetOrDefault flat_decimals 1 1 1 1 (1,1,1) +dictGetOrDefault flat_decimals 0 42 42 42 (42,42,42) +dictGet hashed_decimals 1 1 1 1 (1,1,1) +dictGetOrDefault hashed_decimals 1 1 1 1 (1,1,1) +dictGetOrDefault hashed_decimals 0 42 42 42 (42,42,42) +dictGet cache_decimals 1 1 1 1 (1,1,1) +dictGetOrDefault cache_decimals 1 1 1 1 (1,1,1) +dictGetOrDefault cache_decimals 0 42 42 42 (42,42,42) +dictGet complex_hashed_decimals (1) 1 1 1 (1,1,1) +dictGetOrDefault complex_hashed_decimals (1) 1 1 1 (1,1,1) +dictGetOrDefault complex_hashed_decimals (0) 42 42 42 (42,42,42) +dictGet complex_cache_decimals (1) 1 1 1 (1,1,1) +dictGetOrDefault complex_cache_decimals (1) 1 1 1 (1,1,1) +dictGetOrDefault complex_cache_decimals (0) 42 42 42 (42,42,42) diff --git a/tests/queries/0_stateless/00975_values_list.reference b/tests/queries/0_stateless/00975_values_list.reference index f8ada08d130..d0811d264b0 100644 --- a/tests/queries/0_stateless/00975_values_list.reference +++ b/tests/queries/0_stateless/00975_values_list.reference @@ -10,5 +10,5 @@ cadabra abracadabra 23 23 23 24 24 24 -1.6660 a b +1.666 a b \N diff --git a/tests/queries/0_stateless/00979_toFloat_monotonicity.reference b/tests/queries/0_stateless/00979_toFloat_monotonicity.reference index 7d9895ef9f3..b8ec6976930 100644 --- a/tests/queries/0_stateless/00979_toFloat_monotonicity.reference +++ b/tests/queries/0_stateless/00979_toFloat_monotonicity.reference @@ -2,5 +2,5 @@ 7777 7777 7777 -7777.000 -7777.000 +7777 +7777 diff --git a/tests/queries/0_stateless/00980_crash_nullable_decimal.reference b/tests/queries/0_stateless/00980_crash_nullable_decimal.reference index be6e399c4d9..fcb49fa9945 100644 --- a/tests/queries/0_stateless/00980_crash_nullable_decimal.reference +++ b/tests/queries/0_stateless/00980_crash_nullable_decimal.reference @@ -1,7 +1,7 @@ -1.00 -1.00 -1.00 -1.00 -1.00 -1.00 -1.00 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/01018_empty_aggregation_filling.reference b/tests/queries/0_stateless/01018_empty_aggregation_filling.reference index 4595c3b9112..c29807a7e15 100644 --- a/tests/queries/0_stateless/01018_empty_aggregation_filling.reference +++ b/tests/queries/0_stateless/01018_empty_aggregation_filling.reference @@ -45,7 +45,7 @@ nan \N 0 \N -0.00 +0 \N --- Other Types Non-empty --- hello diff --git a/tests/queries/0_stateless/01055_minmax_index_compact_parts.reference b/tests/queries/0_stateless/01055_minmax_index_compact_parts.reference index 0f5a8eb904e..8bde896b02a 100644 --- a/tests/queries/0_stateless/01055_minmax_index_compact_parts.reference +++ b/tests/queries/0_stateless/01055_minmax_index_compact_parts.reference @@ -1,6 +1,6 @@ -0 5 4.7 6.50 cba b 2014-01-04 -1 5 4.7 6.50 cba b 2014-03-11 -11 5 4.7 6.50 cba b 2014-06-11 -12 5 4.7 6.50 cba b 2015-01-01 +0 5 4.7 6.5 cba b 2014-01-04 +1 5 4.7 6.5 cba b 2014-03-11 +11 5 4.7 6.5 cba b 2014-06-11 +12 5 4.7 6.5 cba b 2015-01-01 "rows_read": 4, "rows_read": 2, diff --git a/tests/queries/0_stateless/01087_storage_generate.reference b/tests/queries/0_stateless/01087_storage_generate.reference index 3680d8d943d..78c6784f7d2 100644 --- a/tests/queries/0_stateless/01087_storage_generate.reference +++ b/tests/queries/0_stateless/01087_storage_generate.reference @@ -4,7 +4,7 @@ [88] 34528.4014 ('2031-12-09 00:40:39.898','9ef777c8-de0e-d25e-e16c-5b624f88523c') [-1] 121968.7945 ('2060-02-05 09:18:12.011','7655e515-d2ca-2f06-0950-e4f44f69aca7') [-103,75] -135033.4349 ('2038-12-19 20:38:58.695','86b57d15-292d-2517-9acf-47cd053e7a3a') -[110] -202668.6900 ('2009-06-18 01:53:29.808','bc630f78-7d58-0c46-dd4b-27fc35625e96') +[110] -202668.69 ('2009-06-18 01:53:29.808','bc630f78-7d58-0c46-dd4b-27fc35625e96') [-22,2] 168636.9728 ('2074-09-03 09:20:20.936','7624ce27-9bff-4e9d-3f18-6851a97dd0ca') [-22,-62] -75192.4989 ('2085-10-11 21:51:12.855','a4c4d0ed-f448-244e-1723-ca1bba816f2b') [-2,-90] 133592.5064 ('2010-10-28 21:18:04.633','8ba9103b-f90c-b49b-38c1-223ae5f42bf7') @@ -25,23 +25,23 @@ [95,38] -65083.7371 ('2015-03-10 13:33:16.429','47bd199c-f99e-51ea-84e9-b65cce9d167c') [91,110,72] 130908.9643 ('2036-03-16 15:17:53.679','0dd4ca31-1e09-d7e0-f3df-60cad3cfa805') [] 208972.3779 ('2034-03-05 22:29:21.994','1069d77c-dfd2-912e-60b8-3c5b964f7e11') -[-32] 167938.5050 ('2093-09-10 20:39:39.050','9d1025b6-2d0c-1d84-dafd-02668eb29270') +[-32] 167938.505 ('2093-09-10 20:39:39.050','9d1025b6-2d0c-1d84-dafd-02668eb29270') [] 153744.6987 ('2088-10-02 11:02:11.024','a88e6cb7-2210-5ce5-6bcf-24afc0eca5b6') -[67] -74220.6650 ('2074-12-30 18:43:40.817','68096065-18c8-8aca-fd21-15330ead669d') +[67] -74220.665 ('2074-12-30 18:43:40.817','68096065-18c8-8aca-fd21-15330ead669d') [6] 66759.8938 ('2091-09-01 19:07:18.219','bb14f4cc-0b54-9a8c-e835-71333b28c03b') [-28,-82,9] 168625.3131 ('2002-03-20 21:02:30.321','405bb877-6e28-8b91-cb62-bd82a3fa797c') -[] -19760.1670 ('2044-11-08 07:52:03.325','13769348-9e58-0e75-3972-8bbadc150715') +[] -19760.167 ('2044-11-08 07:52:03.325','13769348-9e58-0e75-3972-8bbadc150715') [] 160663.7797 ('2025-04-12 13:17:53.501','e6370321-94f5-97e6-0348-a84e72ff5b42') [-17,18] 99105.9856 ('1972-05-01 12:23:11.688','02618b9e-97cd-4698-d2e8-3f52f4c5a09a') [86,77] -116990.3914 ('1981-12-31 05:06:54.198','3ac42bb4-8652-b1a8-10bb-98f0337261f8') [-109,69,-63] -151527.3587 ('2001-01-17 11:19:56.504','77fe7ee2-f279-2855-bfd2-a7d7cee678cc') [] -57762.3928 ('1978-08-16 18:47:37.660','ab9a110a-fd8d-3c4c-5a49-34c2005536ce') [-77] 107274.6407 ('2017-01-12 12:03:02.657','c1ad4f17-cc54-45f3-9410-9c1011653f6d') -[] 107133.6410 ('2050-10-05 06:29:27.154','36e576aa-c77f-994e-1925-4a4c40da3a0f') +[] 107133.641 ('2050-10-05 06:29:27.154','36e576aa-c77f-994e-1925-4a4c40da3a0f') [] 46672.2176 ('2094-01-21 20:25:39.144','e9ba850d-604e-bc7d-417c-1078e89d4615') [-87,-122,-65] -86258.4663 ('2081-06-17 03:37:45.498','64795221-9719-7937-b4d2-be5f30065ece') [-53] -48672.1424 ('1992-06-27 17:27:23.602','7c67bc31-c7bb-6197-fdca-f73329b976f2') -[34] -108954.7820 ('2096-07-03 23:06:30.632','9c1b37d7-4ced-9428-a0ae-34c5436b14c4') +[34] -108954.782 ('2096-07-03 23:06:30.632','9c1b37d7-4ced-9428-a0ae-34c5436b14c4') [] -168124.2364 ('1987-06-03 06:47:12.945','d1c39af4-f920-5095-b8e2-0f878950167b') [] -112431.4799 ('2021-07-26 07:04:58.527','da07a72d-7e1f-8890-4c4b-326835d11b39') [-35,-95,58] -181254.9139 ('2086-11-12 17:17:14.473','22f74d0b-dfc0-3f7a-33f4-8055d8fa7846') @@ -61,10 +61,10 @@ [-35,-58,-101] -9101.5369 ('2023-08-24 20:56:11.695','87fbe3f9-b1f0-c030-a4c0-8662045923b4') [-58,87] 122510.9099 ('2019-08-09 17:40:29.849','c1d3a2cc-878f-c2c3-4a0b-10e98cda8b4a') [4,19,58] -13496.8672 ('2027-05-01 09:11:48.659','8996ae31-d670-cbfe-b735-b16b7c3b3476') -[23,-75,-89] -51218.2860 ('2010-06-02 02:49:03.396','d32b8b61-cc3e-31fa-2a2a-abefa60bfcee') +[23,-75,-89] -51218.286 ('2010-06-02 02:49:03.396','d32b8b61-cc3e-31fa-2a2a-abefa60bfcee') [50] -45297.4315 ('2087-04-15 06:46:08.247','04fe9603-97fc-07a4-6248-0f21e408c884') [-23,17,63] 89185.9462 ('2065-10-26 08:27:12.817','a5fbf764-70b4-8b65-4a8f-7550abca3859') -[-6] -129925.3690 ('2013-11-05 07:44:45.233','11db26b3-e2b5-b9fa-6b0e-79c43a2e67ab') +[-6] -129925.369 ('2013-11-05 07:44:45.233','11db26b3-e2b5-b9fa-6b0e-79c43a2e67ab') [-72,-108] 203171.5475 ('2000-01-28 09:34:58.032','14d5399e-7949-20c7-0e47-85e2fce5836c') [-73,34,-27] 2676.7265 ('2057-10-25 14:37:10.049','00049a92-4350-badb-3764-dd7f019b9b31') [65,-7] -153472.9461 ('1973-04-12 02:34:41.245','e0a0324d-1552-d11e-f3a5-fbd822d206c5') @@ -73,13 +73,13 @@ [107] 9694.1102 ('1984-11-02 13:11:34.034','e973db18-07b7-2117-f3ba-e7002adfa939') [] -76460.9664 ('2051-02-10 09:54:42.143','b8344c22-9e8a-7052-c644-9c3e5989cdf1') [59,59,0] 27041.7606 ('2083-02-17 18:21:22.547','4d6b137b-a3e1-f36d-2c0c-c8d718dda388') -[-114] 133673.9630 ('2005-10-02 20:34:27.452','04785b75-30e5-af8b-547e-d15bcb7f49fb') -[43] -169861.2000 ('2006-12-13 09:26:13.923','cb865d38-d961-d7f9-acbb-583b9f31252f') +[-114] 133673.963 ('2005-10-02 20:34:27.452','04785b75-30e5-af8b-547e-d15bcb7f49fb') +[43] -169861.2 ('2006-12-13 09:26:13.923','cb865d38-d961-d7f9-acbb-583b9f31252f') [] 197115.2174 ('2060-04-08 04:17:00.488','0f26c4b4-b24c-1fd5-c619-31bcf71a4831') [-25] -200081.9506 ('2055-12-25 02:30:16.276','0b32ad69-2c84-4269-9718-e3171482878a') [14,110] -40196.4463 ('2084-08-13 19:37:07.588','ed882071-acba-b3ab-5d77-d79a9544a834') [-62,-71,-82] -154958.9747 ('2100-07-08 02:32:53.741','7711c7c1-0d22-e302-fc86-61ef5e68db96') -[96,-114,-101] 78910.3320 ('2100-07-19 15:02:27.109','756bfd26-c4b3-94b8-e991-c7ab7a833b76') +[96,-114,-101] 78910.332 ('2100-07-19 15:02:27.109','756bfd26-c4b3-94b8-e991-c7ab7a833b76') [49] 80117.2267 ('1970-07-04 03:50:56.748','aebac019-9054-4a77-2ccd-8801fc4a7496') [] 102078.4801 ('2055-01-07 01:22:33.624','21f2e59a-a1ca-5df3-27fd-aa95456cfbe5') [-106] -108728.4237 ('2020-05-27 11:56:18.121','6b7b6674-9342-2360-4cc0-f7ef8a2404de') @@ -91,13 +91,13 @@ [] 212557.3762 ('2069-03-03 07:21:08.439','9e676cac-36e6-2962-f7b1-578214f0dfbd') [-128,55] 80471.0777 ('1970-04-01 18:54:40.257','ca358854-416b-9c95-0b9b-c7fed7bb7cb5') [-30,-54] -132205.4512 ('2017-12-15 22:54:15.750','3558faa4-2d2f-c533-437f-1e03d3600f1d') -[-116,-72] -91499.6670 ('2105-09-23 21:06:17.755','07bb6e47-3234-c268-40d7-332388dc06f8') +[-116,-72] -91499.667 ('2105-09-23 21:06:17.755','07bb6e47-3234-c268-40d7-332388dc06f8') [] -201636.5228 ('2085-01-27 07:54:42.717','86c3bdc3-ff0f-1723-07c2-845aa3c02370') [-103,-39] 44330.7722 ('2064-07-02 11:08:28.068','0869c79d-6bdd-5d2d-a3d1-ffe13f6aa810') [99] -31035.5391 ('2093-07-26 01:50:23.026','aeb59338-254f-dc09-fbd7-263da415e211') [101] 157961.4729 ('2036-05-04 02:35:07.845','8b6221a9-8dad-4655-7460-6b3031b06893') [111] 84732.4403 ('1997-04-06 16:10:18.624','08806a79-59f4-c833-eedc-a200bb851767') -[9,-48] -190491.5590 ('2031-11-03 19:47:03.757','914e6166-c96e-e0e4-101a-0bb516cf5a2f') +[9,-48] -190491.559 ('2031-11-03 19:47:03.757','914e6166-c96e-e0e4-101a-0bb516cf5a2f') [-41] -132501.8311 ('2089-11-21 21:38:28.848','6de6cc8d-3c49-641e-fb12-87ed5ecb97b0') [77] 64903.6579 ('1985-04-17 17:08:03.998','26484b8a-f3f1-587f-7777-bc7a57a689c3') - diff --git a/tests/queries/0_stateless/01087_table_function_generate.reference b/tests/queries/0_stateless/01087_table_function_generate.reference index bf301d34eb3..ea4162e4840 100644 --- a/tests/queries/0_stateless/01087_table_function_generate.reference +++ b/tests/queries/0_stateless/01087_table_function_generate.reference @@ -86,7 +86,7 @@ Decimal(9, 4) Decimal(18, 8) Decimal(18, 8) -18731.5032 81241713112.39967992 -10576027963457111164764.0798899532879521 65289.5061 -27889310937.24180887 5807515838469365530027.7612329616030438 -197586.1517 -751754543.85331084 3835903211857734974086.0358362773591932 -183596.0063 8217353434.41964030 13633006218585943284268.9826084812209912 +183596.0063 8217353434.4196403 13633006218585943284268.9826084812209912 73041.2674 -88881500366.49430454 -148702703925022894263.3187064158377476 101454.4494 -27768337.71540858 -634829280961262229789.4961995996929358 -174012.0101 -13636289325.35403038 -3611949395160064991369.2765012316944096 @@ -226,12 +226,12 @@ RL,{Xs\\tw [124] -114719.5228 ('2010-11-11 22:57:23.722','c1046ffb-3415-cc3a-509a-e0005856d7d7') - [] 1900051923 { -189530.5846 h -5.6279699579452485e47 ('1984-12-06','2028-08-17 06:05:01','2036-04-02 23:52:28.468','4b3d498c-dd44-95c1-5b75-921504ec5d8d') F743 -[-102,-118] 392272782 Eb -14818.0200 o -2.664492247169164e59 ('2082-12-26','2052-09-09 06:50:50','2088-04-21 05:07:08.245','aeb9c26e-0ee7-2b8e-802b-2a96319b8e60') CBF4 +[-102,-118] 392272782 Eb -14818.02 o -2.664492247169164e59 ('2082-12-26','2052-09-09 06:50:50','2088-04-21 05:07:08.245','aeb9c26e-0ee7-2b8e-802b-2a96319b8e60') CBF4 [-71] 775049089 \N -158115.1178 w 4.1323844687113747e-305 ('2108-04-19','2090-07-31 16:45:26','2076-07-10 09:11:06.385','57c69bc6-dddd-0975-e932-a7b5173a1304') EB1D [-28,100] 3675466147 { -146685.1749 h 3.6676044396877755e142 ('2017-10-25','2100-02-28 18:07:18','2055-10-14 06:36:20.056','14949dae-dfa8-a124-af83-887348b2f609') 6D88 [-23] 2514120753 (`u, -119659.6174 w 1.3231258347475906e34 ('2141-04-06','2074-08-10 06:25:12','1976-12-04 18:31:55.745','86a9b3c1-4593-4d56-7762-3aa1dd22cbbf') AD43 [11,-36] 3308237300 \N 171205.1896 \N 5.634708707075817e195 ('1974-10-31','1993-12-24 09:38:45','2038-07-15 05:22:51.805','63d999b8-8cca-e237-c4a4-4dd7d0096f65') 609E -[39] 1614362420 `4A8P 157144.0630 o -1.1843143253872814e-255 ('2147-08-18','2072-09-28 18:27:27','2073-07-10 12:19:58.146','6483f5c0-8733-364c-4fa0-9948d32e8903') A886 +[39] 1614362420 `4A8P 157144.063 o -1.1843143253872814e-255 ('2147-08-18','2072-09-28 18:27:27','2073-07-10 12:19:58.146','6483f5c0-8733-364c-4fa0-9948d32e8903') A886 [48,-120] 3848918261 1 Date: Sun, 15 Aug 2021 10:54:30 +0300 Subject: [PATCH 057/220] Update more tests --- tests/queries/0_stateless/01273_arrow_decimal.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01273_arrow_decimal.reference b/tests/queries/0_stateless/01273_arrow_decimal.reference index a512796de07..1358d3fa841 100644 --- a/tests/queries/0_stateless/01273_arrow_decimal.reference +++ b/tests/queries/0_stateless/01273_arrow_decimal.reference @@ -1,2 +1,2 @@ -0.1230 0.12312312 0.1231231231230000 0.12312312312312312300000000000000 -0.1230 0.12312312 0.1231231231230000 0.12312312312312312300000000000000 +0.123 0.12312312 0.123123123123 0.123123123123123123 +0.123 0.12312312 0.123123123123 0.123123123123123123 From 8fe3aa6cef9ec42f2e8907fb1aeaee197473ddce Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 15 Aug 2021 11:28:08 +0300 Subject: [PATCH 058/220] Update 01236_graphite_mt.sql --- tests/queries/0_stateless/01236_graphite_mt.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/01236_graphite_mt.sql b/tests/queries/0_stateless/01236_graphite_mt.sql index 552e29082d4..a6dd4b8b6fb 100644 --- a/tests/queries/0_stateless/01236_graphite_mt.sql +++ b/tests/queries/0_stateless/01236_graphite_mt.sql @@ -23,7 +23,7 @@ WITH dates AS select 2, 'max_2', today - number * 60 - 30, number, 1, number from dates, numbers(300) union all -- Older than 2 days use 6000 second windows - select 1 ASK key, 'sum_1' AS s, older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all + select 1 AS key, 'sum_1' AS s, older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all select 2, 'sum_1', older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all select 1, 'sum_2', older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all select 2, 'sum_2', older_date - number * 60 - 30, number, 1, number from dates, numbers(1200) union all From e24087526b6bc9b5524806c6e98563e7cc73da25 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 15 Aug 2021 11:59:53 +0300 Subject: [PATCH 059/220] Update test --- .../0_stateless/01658_read_file_to_stringcolumn.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference index 1d0901cf9f6..579c0dcb270 100644 --- a/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference +++ b/tests/queries/0_stateless/01658_read_file_to_stringcolumn.reference @@ -13,13 +13,13 @@ ccccccccc :35 :35 :233 -699415 +695071 aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb ccccccccc aaaaaaaaa bbbbbbbbb -699415 0 +695071 0 :0 :70 :79 From 71082b8656fb3cf28a158399daffb4d90ad63f92 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Sun, 15 Aug 2021 13:28:13 +0300 Subject: [PATCH 060/220] Update 00597_push_down_predicate_long.sql --- tests/queries/0_stateless/00597_push_down_predicate_long.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql index 412b8b7852c..8096cbef46b 100644 --- a/tests/queries/0_stateless/00597_push_down_predicate_long.sql +++ b/tests/queries/0_stateless/00597_push_down_predicate_long.sql @@ -9,7 +9,7 @@ CREATE TABLE test_00597(date Date, id Int8, name String, value Int64) ENGINE = M CREATE VIEW test_view_00597 AS SELECT * FROM test_00597; -- TODO: This query should execute successfully: -SELECT * FROM (SELECT floor(floor(1, floor(NULL), id = 257), floor(floor(floor(floor(NULL), '10485.76', '9223372036854775807', NULL), floor(10, floor(65535, NULL), 100.0000991821289), NULL)), '2.56'), b.* FROM (SELECT floor(floor(floor(floor(NULL), 1000.0001220703125))), * FROM test_00597) AS b) WHERE id = 257; -- { serverError 96 } +SELECT * FROM (SELECT floor(floor(1, floor(NULL), id = 257), floor(floor(floor(floor(NULL), '10485.76', '9223372036854775807', NULL), floor(10, floor(65535, NULL), 100.0000991821289), NULL)), '2.56'), b.* FROM (SELECT floor(floor(floor(floor(NULL), 1000.0001220703125))), * FROM test_00597) AS b) WHERE id = 257; -- { serverError 352 } INSERT INTO test_00597 VALUES('2000-01-01', 1, 'test string 1', 1); INSERT INTO test_00597 VALUES('2000-01-01', 2, 'test string 2', 2); From b9d8ee125b52bc9f41862e888072d874b5002bd5 Mon Sep 17 00:00:00 2001 From: jasine Date: Sun, 15 Aug 2021 22:26:40 +0800 Subject: [PATCH 061/220] feat: add conversion between snowflake id and datetime --- src/Functions/FunctionSnowflake.h | 208 +++++++++++++++++++ src/Functions/dateTime64ToSnowflake.cpp | 14 ++ src/Functions/dateTimeToSnowflake.cpp | 14 ++ src/Functions/registerFunctions.cpp | 2 + src/Functions/registerFunctionsSnowflake.cpp | 22 ++ src/Functions/snowflakeToDateTime.cpp | 14 ++ src/Functions/snowflakeToDateTime64.cpp | 14 ++ 7 files changed, 288 insertions(+) create mode 100644 src/Functions/FunctionSnowflake.h create mode 100644 src/Functions/dateTime64ToSnowflake.cpp create mode 100644 src/Functions/dateTimeToSnowflake.cpp create mode 100644 src/Functions/registerFunctionsSnowflake.cpp create mode 100644 src/Functions/snowflakeToDateTime.cpp create mode 100644 src/Functions/snowflakeToDateTime64.cpp diff --git a/src/Functions/FunctionSnowflake.h b/src/Functions/FunctionSnowflake.h new file mode 100644 index 00000000000..cf3a91b8e69 --- /dev/null +++ b/src/Functions/FunctionSnowflake.h @@ -0,0 +1,208 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; +} + + +class FunctionDateTimeToSnowflake : public IFunction +{ +private: + const char * name; +public: + FunctionDateTimeToSnowflake( const char * name_) + : name(name_) + { + } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isVariadic() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (!isDateTime(arguments[0].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The only argument for function {} must be DateTime", name); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src = arguments[0]; + const auto & col = *src.column; + + auto res_column = ColumnInt64::create(input_rows_count); + auto & result_data = res_column->getData(); + + const auto & source_data = typeid_cast(col).getData(); + for (size_t i = 0; i < input_rows_count; ++i) + { + result_data[i] = (int64_t(source_data[i])*1000-1288834974657)<<22; + } + + return res_column; + } +}; + + +class FunctionSnowflakeToDateTime : public IFunction +{ +private: + const char * name; +public: + FunctionSnowflakeToDateTime(const char * name_) + : name(name_) + { + } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + + if (arguments.size() < 1 || arguments.size() > 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); + + if (!typeid_cast(arguments[0].type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The first argument for function {} must be Int64", name); + + std::string timezone; + if (arguments.size() == 2) + timezone = extractTimeZoneNameFromFunctionArguments(arguments, 1, 0); + + return std::make_shared(timezone); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src = arguments[0]; + const auto & col = *src.column; + + auto res_column = ColumnUInt32::create(input_rows_count); + auto & result_data = res_column->getData(); + + const auto & source_data = typeid_cast(col).getData(); + + for (size_t i = 0; i < input_rows_count; ++i) + { + result_data[i] = ((source_data[i]>>22)+1288834974657)/1000; + } + + return res_column; + } +}; + + +class FunctionDateTime64ToSnowflake : public IFunction +{ +private: + const char * name; +public: + FunctionDateTime64ToSnowflake( const char * name_) + : name(name_) + { + } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 1; } + bool isVariadic() const override { return false; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (!isDateTime64(arguments[0].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The only argument for function {} must be DateTime64", name); + + return std::make_shared(); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src = arguments[0]; + const auto & col = *src.column; + + auto res_column = ColumnInt64::create(input_rows_count); + auto & result_data = res_column->getData(); + + const auto & source_data = typeid_cast &>(col).getData(); + for (size_t i = 0; i < input_rows_count; ++i) + { + result_data[i] = (source_data[i]-1288834974657)<<22; + } + + return res_column; + } +}; + + +class FunctionSnowflakeToDateTime64 : public IFunction +{ +private: + const char * name; +public: + FunctionSnowflakeToDateTime64(const char * name_) + : name(name_) + { + } + + String getName() const override { return name; } + size_t getNumberOfArguments() const override { return 0; } + bool isVariadic() const override { return true; } + bool useDefaultImplementationForConstants() const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + + if (arguments.size() < 1 || arguments.size() > 2) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); + + if (!typeid_cast(arguments[0].type.get())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The first argument for function {} must be Int64", name); + + std::string timezone; + if (arguments.size() == 2) + timezone = extractTimeZoneNameFromFunctionArguments(arguments, 1, 0); + + return std::make_shared(3, timezone); + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const auto & src = arguments[0]; + const auto & col = *src.column; + + auto res_column = ColumnDecimal::create(input_rows_count, 3); + auto & result_data = res_column->getData(); + + const auto & source_data = typeid_cast(col).getData(); + + for (size_t i = 0; i < input_rows_count; ++i) + { + result_data[i] = (source_data[i]>>22)+1288834974657; + } + + return res_column; + } +}; + +} diff --git a/src/Functions/dateTime64ToSnowflake.cpp b/src/Functions/dateTime64ToSnowflake.cpp new file mode 100644 index 00000000000..87e35c25371 --- /dev/null +++ b/src/Functions/dateTime64ToSnowflake.cpp @@ -0,0 +1,14 @@ +#include +#include + +namespace DB +{ + +void registerDateTime64ToSnowflake(FunctionFactory & factory) +{ + factory.registerFunction("dateTime64ToSnowflake", + [](ContextPtr){ return std::make_unique( + std::make_shared("dateTime64ToSnowflake")); }); +} + +} diff --git a/src/Functions/dateTimeToSnowflake.cpp b/src/Functions/dateTimeToSnowflake.cpp new file mode 100644 index 00000000000..246f35cc1dc --- /dev/null +++ b/src/Functions/dateTimeToSnowflake.cpp @@ -0,0 +1,14 @@ +#include +#include + +namespace DB +{ + +void registerDateTimeToSnowflake(FunctionFactory & factory) +{ + factory.registerFunction("dateTimeToSnowflake", + [](ContextPtr){ return std::make_unique( + std::make_shared("dateTimeToSnowflake")); }); +} + +} diff --git a/src/Functions/registerFunctions.cpp b/src/Functions/registerFunctions.cpp index 7e8f35bc0c4..9b1a7faebbe 100644 --- a/src/Functions/registerFunctions.cpp +++ b/src/Functions/registerFunctions.cpp @@ -51,6 +51,7 @@ void registerFunctionBitHammingDistance(FunctionFactory & factory); void registerFunctionTupleHammingDistance(FunctionFactory & factory); void registerFunctionsStringHash(FunctionFactory & factory); void registerFunctionValidateNestedArraySizes(FunctionFactory & factory); +void registerFunctionsSnowflake(FunctionFactory & factory); #if !defined(ARCADIA_BUILD) void registerFunctionBayesAB(FunctionFactory &); #endif @@ -115,6 +116,7 @@ void registerFunctions() registerFunctionTupleHammingDistance(factory); registerFunctionsStringHash(factory); registerFunctionValidateNestedArraySizes(factory); + registerFunctionsSnowflake(factory); #if !defined(ARCADIA_BUILD) registerFunctionBayesAB(factory); diff --git a/src/Functions/registerFunctionsSnowflake.cpp b/src/Functions/registerFunctionsSnowflake.cpp new file mode 100644 index 00000000000..f0c2feddfb5 --- /dev/null +++ b/src/Functions/registerFunctionsSnowflake.cpp @@ -0,0 +1,22 @@ +namespace DB +{ + +class FunctionFactory; + +void registerDateTimeToSnowflake(FunctionFactory &); +void registerSnowflakeToDateTime(FunctionFactory &); + +void registerDateTime64ToSnowflake(FunctionFactory &); +void registerSnowflakeToDateTime64(FunctionFactory &); + + +void registerFunctionsSnowflake(FunctionFactory & factory) +{ + registerDateTimeToSnowflake(factory); + registerSnowflakeToDateTime(factory); + + registerDateTime64ToSnowflake(factory); + registerSnowflakeToDateTime64(factory); +} + +} diff --git a/src/Functions/snowflakeToDateTime.cpp b/src/Functions/snowflakeToDateTime.cpp new file mode 100644 index 00000000000..37f5e07512f --- /dev/null +++ b/src/Functions/snowflakeToDateTime.cpp @@ -0,0 +1,14 @@ +#include +#include + +namespace DB +{ + +void registerSnowflakeToDateTime(FunctionFactory & factory) +{ + factory.registerFunction("snowflakeToDateTime", + [](ContextPtr){ return std::make_unique( + std::make_shared("snowflakeToDateTime")); }); +} + +} diff --git a/src/Functions/snowflakeToDateTime64.cpp b/src/Functions/snowflakeToDateTime64.cpp new file mode 100644 index 00000000000..ef9502a224e --- /dev/null +++ b/src/Functions/snowflakeToDateTime64.cpp @@ -0,0 +1,14 @@ +#include +#include + +namespace DB +{ + +void registerSnowflakeToDateTime64(FunctionFactory & factory) +{ + factory.registerFunction("snowflakeToDateTime64", + [](ContextPtr){ return std::make_unique( + std::make_shared("snowflakeToDateTime64")); }); +} + +} From aa49f76bf0184fb2e4abe9487df7f7c88f8344d4 Mon Sep 17 00:00:00 2001 From: jasine Date: Mon, 16 Aug 2021 00:49:33 +0800 Subject: [PATCH 062/220] fix: style --- src/Functions/FunctionSnowflake.h | 11 +++++------ src/Functions/registerFunctionsSnowflake.cpp | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Functions/FunctionSnowflake.h b/src/Functions/FunctionSnowflake.h index cf3a91b8e69..3dd378e4956 100644 --- a/src/Functions/FunctionSnowflake.h +++ b/src/Functions/FunctionSnowflake.h @@ -25,7 +25,7 @@ class FunctionDateTimeToSnowflake : public IFunction private: const char * name; public: - FunctionDateTimeToSnowflake( const char * name_) + FunctionDateTimeToSnowflake(const char * name_) : name(name_) { } @@ -79,7 +79,6 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() < 1 || arguments.size() > 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); @@ -107,7 +106,7 @@ public: { result_data[i] = ((source_data[i]>>22)+1288834974657)/1000; } - + return res_column; } }; @@ -118,7 +117,7 @@ class FunctionDateTime64ToSnowflake : public IFunction private: const char * name; public: - FunctionDateTime64ToSnowflake( const char * name_) + FunctionDateTime64ToSnowflake(const char * name_) : name(name_) { } @@ -172,7 +171,7 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - + if (arguments.size() < 1 || arguments.size() > 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); @@ -200,7 +199,7 @@ public: { result_data[i] = (source_data[i]>>22)+1288834974657; } - + return res_column; } }; diff --git a/src/Functions/registerFunctionsSnowflake.cpp b/src/Functions/registerFunctionsSnowflake.cpp index f0c2feddfb5..7a0569ee16a 100644 --- a/src/Functions/registerFunctionsSnowflake.cpp +++ b/src/Functions/registerFunctionsSnowflake.cpp @@ -14,7 +14,7 @@ void registerFunctionsSnowflake(FunctionFactory & factory) { registerDateTimeToSnowflake(factory); registerSnowflakeToDateTime(factory); - + registerDateTime64ToSnowflake(factory); registerSnowflakeToDateTime64(factory); } From d5db8f89796ae4ea28c434a2aca6078f84c3b9bd Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Sun, 15 Aug 2021 20:33:16 +0300 Subject: [PATCH 063/220] Update docs/en/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index 1d037382717..ce9cfc8490f 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -17,7 +17,7 @@ Columns with request parameters: - `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the request. - `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — Host port. - `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — The session ID that the ZooKeeper server sets for each connection. -- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. Usually, it is just a sequential request number. It is the same for the request line and the paired `response`/`finalize` line. +- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request line and the paired `response`/`finalize` line. - `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The request whether the [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches) has been installed. - `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — The request or response type. - `path` ([String](../../sql-reference/data-types/string.md)) — The path to the ZooKeeper node specified in the request (if the request requires specifying a path) or an empty string. From 8a79ed0d38a4601ab7180b1c59bdb9648a537f42 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Sun, 15 Aug 2021 20:34:03 +0300 Subject: [PATCH 064/220] Update docs/en/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index ce9cfc8490f..a9e1fefffea 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -19,7 +19,7 @@ Columns with request parameters: - `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — The session ID that the ZooKeeper server sets for each connection. - `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request line and the paired `response`/`finalize` line. - `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The request whether the [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches) has been installed. -- `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — The request or response type. +- `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — The type of request or response. - `path` ([String](../../sql-reference/data-types/string.md)) — The path to the ZooKeeper node specified in the request (if the request requires specifying a path) or an empty string. - `data` ([String](../../sql-reference/data-types/string.md)) — The data written to the ZooKeeper node (for the `SET` and `CREATE` requests — what the request wanted to write, for the response to the `GET` request — what was read) or an empty string. - `is_ephemeral` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [ephemeral](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Ephemeral+Nodes). From 4c9a5aa0f1eba974826e1739e80b2c7bf848e11b Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Sun, 15 Aug 2021 20:34:23 +0300 Subject: [PATCH 065/220] Update docs/en/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index a9e1fefffea..84d5465fb4f 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -20,7 +20,7 @@ Columns with request parameters: - `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request line and the paired `response`/`finalize` line. - `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The request whether the [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches) has been installed. - `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — The type of request or response. -- `path` ([String](../../sql-reference/data-types/string.md)) — The path to the ZooKeeper node specified in the request (if the request requires specifying a path) or an empty string. +- `path` ([String](../../sql-reference/data-types/string.md)) — The path to the ZooKeeper node specified in the request, or an empty string if the request not requires specifying a path. - `data` ([String](../../sql-reference/data-types/string.md)) — The data written to the ZooKeeper node (for the `SET` and `CREATE` requests — what the request wanted to write, for the response to the `GET` request — what was read) or an empty string. - `is_ephemeral` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [ephemeral](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Ephemeral+Nodes). - `is_sequential` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [sequential](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Sequence+Nodes+--+Unique+Naming). From 6b77dcacca3c937a9ca0fdd8fe6b93445f5fdda6 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Sun, 15 Aug 2021 20:34:46 +0300 Subject: [PATCH 066/220] Update docs/en/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index 84d5465fb4f..0374c406854 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -24,7 +24,7 @@ Columns with request parameters: - `data` ([String](../../sql-reference/data-types/string.md)) — The data written to the ZooKeeper node (for the `SET` and `CREATE` requests — what the request wanted to write, for the response to the `GET` request — what was read) or an empty string. - `is_ephemeral` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [ephemeral](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Ephemeral+Nodes). - `is_sequential` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [sequential](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Sequence+Nodes+--+Unique+Naming). -- `version` ([Nullable(Int32)](../../sql-reference/data-types/nullable.md)) — The version of the ZooKeeper node that the request expects when executing (for `CHECK`, `SET`, `REMOVE` requests; `-1` if the request does not check the version) or NULL for other requests that do not support version checking. +- `version` ([Nullable(Int32)](../../sql-reference/data-types/nullable.md)) — The version of the ZooKeeper node that the request expects when executing. This is supported for `CHECK`, `SET`, `REMOVE` requests (is relevant `-1` if the request does not check the version or `NULL` for other requests that do not support version checking). - `requests_size` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of requests included in the "multi" request (this is a special request that consists of several consecutive ordinary requests and executes them atomically). All requests included in "multi" request will have the same `xid`. - `request_idx` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of the request included in multi (for multi — `0`, then in order from `1`). From 382491b7a0ad139cf6a5eb45d7d44788d9aaf458 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Sun, 15 Aug 2021 20:34:57 +0300 Subject: [PATCH 067/220] Update docs/en/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index 0374c406854..c718d7013f4 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -37,7 +37,7 @@ Columns with request response parameters: - `ZOPERATIONTIMEOUT` — The request execution timeout has expired. - `ZSESSIONEXPIRED` — The session has expired. - `NULL` — The request is completed. -- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: NULL. +- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. - `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: NULL. - `path_created` ([String](../../sql-reference/data-types/string.md)) — The path to the created ZooKeeper node (for responses to the `CREATE` request), may differ from the `path` if the node is created as a sequential. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that caused this ZooKeeper node to be created. From 388db2551611a83fa85f8192870be5e7f8172242 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Sun, 15 Aug 2021 20:35:04 +0300 Subject: [PATCH 068/220] Update docs/en/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index c718d7013f4..07bd321ccc5 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -38,7 +38,7 @@ Columns with request response parameters: - `ZSESSIONEXPIRED` — The session has expired. - `NULL` — The request is completed. - `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. -- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: NULL. +- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. - `path_created` ([String](../../sql-reference/data-types/string.md)) — The path to the created ZooKeeper node (for responses to the `CREATE` request), may differ from the `path` if the node is created as a sequential. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that caused this ZooKeeper node to be created. - `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that last modified this ZooKeeper node. From 4ac49fe5b0d4f8b43d4d4717ed8748eee12e4799 Mon Sep 17 00:00:00 2001 From: sevirov <72220289+sevirov@users.noreply.github.com> Date: Sun, 15 Aug 2021 20:35:21 +0300 Subject: [PATCH 069/220] Update docs/en/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index 07bd321ccc5..7e24da82e09 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -4,7 +4,7 @@ The table does not exist if ZooKeeper is not configured. This table contains information about the parameters of the request to the ZooKeeper client and the response from it. -For requests, only columns with request parameters are filled in, and the remaining columns are filled with default values (`0` or NULL). When the response arrives, the data from the response is added to the other columns. +For requests, only columns with request parameters are filled in, and the remaining columns are filled with default values (`0` or `NULL`). When the response arrives, the data from the response is added to the other columns. Columns with request parameters: From 28027c732318bb9192e2ccb915adc544e0561e39 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 16 Aug 2021 05:00:39 +0300 Subject: [PATCH 070/220] Fix some tests --- src/IO/WriteHelpers.h | 10 +++++----- .../test_postgresql_replica_database_engine/test.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 891a8d1f073..feafe87d47d 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -910,17 +910,17 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) { if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); return; } } @@ -928,12 +928,12 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) { if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); return; } } diff --git a/tests/integration/test_postgresql_replica_database_engine/test.py b/tests/integration/test_postgresql_replica_database_engine/test.py index 3763b503b60..40324089b1b 100644 --- a/tests/integration/test_postgresql_replica_database_engine/test.py +++ b/tests/integration/test_postgresql_replica_database_engine/test.py @@ -258,7 +258,7 @@ def test_different_data_types(started_cluster): check_tables_are_synchronized('test_data_types', 'id'); result = instance.query('SELECT * FROM test_database.test_data_types ORDER BY id LIMIT 1;') - assert(result == '0\t-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12.012345\t2000-05-12\t0.20000\t0.20000\n') + assert(result == '0\t-32768\t-2147483648\t-9223372036854775808\t1.12345\t1.123456789\t2147483647\t9223372036854775807\t2000-05-12 12:12:12.012345\t2000-05-12\t0.2\t0.2\n') for i in range(10): col = random.choice(['a', 'b', 'c']) From bdab932f972201e647e69763153f84dd1f9a56df Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 16 Aug 2021 08:56:42 +0300 Subject: [PATCH 071/220] Fix some tests --- src/IO/WriteHelpers.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index feafe87d47d..97fd7b77ba6 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -910,17 +910,17 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) { if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); + writeDecimalFractional(static_cast(x), scale, ostr); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); + writeDecimalFractional(static_cast(x), scale, ostr); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); + writeDecimalFractional(static_cast(x), scale, ostr); return; } } @@ -928,17 +928,17 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) { if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); + writeDecimalFractional(static_cast(x), scale, ostr); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), std::min(scale, std::numeric_limits::digits10), ostr); + writeDecimalFractional(static_cast(x), scale, ostr); return; } } - constexpr size_t max_digits = std::numeric_limits::digits10; + constexpr size_t max_digits = std::numeric_limits::digits10; assert(scale <= max_digits); char buf[max_digits]; memset(buf, '0', scale); From 8adaef7c8ef23647b10cc63091797be2488a522d Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 16 Aug 2021 11:03:23 +0300 Subject: [PATCH 072/220] Make text format for Decimal tuneable --- src/Common/FieldVisitorDump.cpp | 2 +- src/Common/FieldVisitorToString.cpp | 2 +- src/Core/Field.cpp | 108 ++++++++++++------ src/Core/Field.h | 4 +- src/Core/Settings.h | 1 + .../Serializations/SerializationDecimal.cpp | 4 +- src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSettings.h | 1 + src/Formats/ProtobufSerializer.cpp | 6 +- src/Functions/FunctionsConversion.h | 2 +- src/IO/WriteHelpers.h | 20 ++-- 11 files changed, 94 insertions(+), 57 deletions(-) diff --git a/src/Common/FieldVisitorDump.cpp b/src/Common/FieldVisitorDump.cpp index 5e767cf30c1..660677404ad 100644 --- a/src/Common/FieldVisitorDump.cpp +++ b/src/Common/FieldVisitorDump.cpp @@ -20,7 +20,7 @@ template static inline void writeQuoted(const DecimalField & x, WriteBuffer & buf) { writeChar('\'', buf); - writeText(x.getValue(), x.getScale(), buf); + writeText(x.getValue(), x.getScale(), buf, {}); writeChar('\'', buf); } diff --git a/src/Common/FieldVisitorToString.cpp b/src/Common/FieldVisitorToString.cpp index 74dfc55e1db..b8750d95e5e 100644 --- a/src/Common/FieldVisitorToString.cpp +++ b/src/Common/FieldVisitorToString.cpp @@ -26,7 +26,7 @@ template static inline void writeQuoted(const DecimalField & x, WriteBuffer & buf) { writeChar('\'', buf); - writeText(x.getValue(), x.getScale(), buf); + writeText(x.getValue(), x.getScale(), buf, {}); writeChar('\'', buf); } diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index b7b03951ac9..8739f56d991 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -23,65 +23,99 @@ inline Field getBinaryValue(UInt8 type, ReadBuffer & buf) { switch (type) { - case Field::Types::Null: { - return DB::Field(); + case Field::Types::Null: + { + return Field(); } - case Field::Types::UInt64: { + case Field::Types::UInt64: + { UInt64 value; - DB::readVarUInt(value, buf); + readVarUInt(value, buf); return value; } - case Field::Types::UInt128: { + case Field::Types::UInt128: + { UInt128 value; - DB::readBinary(value, buf); + readBinary(value, buf); return value; } - case Field::Types::Int64: { + case Field::Types::UInt256: + { + UInt256 value; + readBinary(value, buf); + return value; + } + case Field::Types::UUID: + { + UUID value; + readBinary(value, buf); + return value; + } + case Field::Types::Int64: + { Int64 value; - DB::readVarInt(value, buf); + readVarInt(value, buf); return value; } - case Field::Types::Float64: { + case Field::Types::Int128: + { + Int128 value; + readBinary(value, buf); + return value; + } + case Field::Types::Int256: + { + Int256 value; + readBinary(value, buf); + return value; + } + case Field::Types::Float64: + { Float64 value; - DB::readFloatBinary(value, buf); + readFloatBinary(value, buf); return value; } - case Field::Types::String: { + case Field::Types::String: + { std::string value; - DB::readStringBinary(value, buf); + readStringBinary(value, buf); return value; } - case Field::Types::Array: { + case Field::Types::Array: + { Array value; - DB::readBinary(value, buf); + readBinary(value, buf); return value; } - case Field::Types::Tuple: { + case Field::Types::Tuple: + { Tuple value; - DB::readBinary(value, buf); + readBinary(value, buf); return value; } - case Field::Types::Map: { + case Field::Types::Map: + { Map value; - DB::readBinary(value, buf); + readBinary(value, buf); return value; } - case Field::Types::AggregateFunctionState: { + case Field::Types::AggregateFunctionState: + { AggregateFunctionStateData value; - DB::readStringBinary(value.name, buf); - DB::readStringBinary(value.data, buf); + readStringBinary(value.name, buf); + readStringBinary(value.data, buf); return value; } } - return DB::Field(); + return Field(); } void readBinary(Array & x, ReadBuffer & buf) { size_t size; UInt8 type; - DB::readBinary(type, buf); - DB::readBinary(size, buf); + readBinary(type, buf); + readBinary(size, buf); for (size_t index = 0; index < size; ++index) x.push_back(getBinaryValue(type, buf)); @@ -93,8 +127,8 @@ void writeBinary(const Array & x, WriteBuffer & buf) size_t size = x.size(); if (size) type = x.front().getType(); - DB::writeBinary(type, buf); - DB::writeBinary(size, buf); + writeBinary(type, buf); + writeBinary(size, buf); for (const auto & elem : x) Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem); @@ -102,19 +136,19 @@ void writeBinary(const Array & x, WriteBuffer & buf) void writeText(const Array & x, WriteBuffer & buf) { - DB::String res = applyVisitor(FieldVisitorToString(), DB::Field(x)); + String res = applyVisitor(FieldVisitorToString(), Field(x)); buf.write(res.data(), res.size()); } void readBinary(Tuple & x, ReadBuffer & buf) { size_t size; - DB::readBinary(size, buf); + readBinary(size, buf); for (size_t index = 0; index < size; ++index) { UInt8 type; - DB::readBinary(type, buf); + readBinary(type, buf); x.push_back(getBinaryValue(type, buf)); } } @@ -122,30 +156,30 @@ void readBinary(Tuple & x, ReadBuffer & buf) void writeBinary(const Tuple & x, WriteBuffer & buf) { const size_t size = x.size(); - DB::writeBinary(size, buf); + writeBinary(size, buf); for (const auto & elem : x) { const UInt8 type = elem.getType(); - DB::writeBinary(type, buf); + writeBinary(type, buf); Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem); } } void writeText(const Tuple & x, WriteBuffer & buf) { - writeFieldText(DB::Field(x), buf); + writeFieldText(Field(x), buf); } void readBinary(Map & x, ReadBuffer & buf) { size_t size; - DB::readBinary(size, buf); + readBinary(size, buf); for (size_t index = 0; index < size; ++index) { UInt8 type; - DB::readBinary(type, buf); + readBinary(type, buf); x.push_back(getBinaryValue(type, buf)); } } @@ -153,19 +187,19 @@ void readBinary(Map & x, ReadBuffer & buf) void writeBinary(const Map & x, WriteBuffer & buf) { const size_t size = x.size(); - DB::writeBinary(size, buf); + writeBinary(size, buf); for (const auto & elem : x) { const UInt8 type = elem.getType(); - DB::writeBinary(type, buf); + writeBinary(type, buf); Field::dispatch([&buf] (const auto & value) { FieldVisitorWriteBinary()(value, buf); }, elem); } } void writeText(const Map & x, WriteBuffer & buf) { - writeFieldText(DB::Field(x), buf); + writeFieldText(Field(x), buf); } template diff --git a/src/Core/Field.h b/src/Core/Field.h index 744675d6e86..0023497e970 100644 --- a/src/Core/Field.h +++ b/src/Core/Field.h @@ -974,9 +974,9 @@ __attribute__ ((noreturn)) inline void writeText(const AggregateFunctionStateDat } template -inline void writeText(const DecimalField & value, WriteBuffer & buf) +inline void writeText(const DecimalField & value, WriteBuffer & buf, bool trailing_zeros = false) { - writeText(value.getValue(), value.getScale(), buf); + writeText(value.getValue(), value.getScale(), buf, trailing_zeros); } template diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 19f9f2a94c8..f2f0c946b82 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -560,6 +560,7 @@ class IColumn; M(UInt64, output_format_avro_sync_interval, 16 * 1024, "Sync interval in bytes.", 0) \ M(Bool, output_format_tsv_crlf_end_of_line, false, "If it is set true, end of line in TSV format will be \\r\\n instead of \\n.", 0) \ M(String, output_format_tsv_null_representation, "\\N", "Custom NULL representation in TSV format", 0) \ + M(Bool, output_format_decimal_trailing_zeros, false, "Output trailing zeros when printing Decimal values. E.g. 1.230000 instead of 1.23.", 0) \ \ M(UInt64, input_format_allow_errors_num, 0, "Maximum absolute amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ M(Float, input_format_allow_errors_ratio, 0, "Maximum relative amount of errors while reading text formats (like CSV, TSV). In case of error, if at least absolute or relative amount of errors is lower than corresponding value, will skip until next line and continue.", 0) \ diff --git a/src/DataTypes/Serializations/SerializationDecimal.cpp b/src/DataTypes/Serializations/SerializationDecimal.cpp index e0073c80aca..88c6d970980 100644 --- a/src/DataTypes/Serializations/SerializationDecimal.cpp +++ b/src/DataTypes/Serializations/SerializationDecimal.cpp @@ -44,10 +44,10 @@ void SerializationDecimal::readText(T & x, ReadBuffer & istr, UInt32 precisio } template -void SerializationDecimal::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings &) const +void SerializationDecimal::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { T value = assert_cast(column).getData()[row_num]; - writeText(value, this->scale, ostr); + writeText(value, this->scale, ostr, settings.decimal_trailing_zeros); } template diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 7b2aac78067..95270cb304f 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -81,6 +81,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.json.quote_64bit_integers = settings.output_format_json_quote_64bit_integers; format_settings.json.quote_denormals = settings.output_format_json_quote_denormals; format_settings.null_as_default = settings.input_format_null_as_default; + format_settings.decimal_trailing_zeros = settings.output_format_decimal_trailing_zeros; format_settings.parquet.row_group_size = settings.output_format_parquet_row_group_size; format_settings.parquet.import_nested = settings.input_format_parquet_import_nested; format_settings.pretty.charset = settings.output_format_pretty_grid_charset.toString() == "ASCII" ? FormatSettings::Pretty::Charset::ASCII : FormatSettings::Pretty::Charset::UTF8; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index d77a7c95d69..3e1e00584c0 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -28,6 +28,7 @@ struct FormatSettings bool write_statistics = true; bool import_nested_json = false; bool null_as_default = true; + bool decimal_trailing_zeros = false; enum class DateTimeInputFormat { diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index c5781ee6c9f..baeefa8f98e 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -1239,7 +1239,7 @@ namespace else { WriteBufferFromOwnString buf; - writeText(decimal, scale, buf); + writeText(decimal, scale, buf, false); cannotConvertValue(buf.str(), TypeName, field_descriptor.type_name()); } }; @@ -1316,9 +1316,9 @@ namespace { WriteBufferFromString buf{str}; if constexpr (std::is_same_v) - writeDateTimeText(decimal, scale, buf); + writeDateTimeText(decimal, scale, buf); else - writeText(decimal, scale, buf); + writeText(decimal, scale, buf, false); } DecimalType stringToDecimal(const String & str) const diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index e57998e4a72..8f34abc0058 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -733,7 +733,7 @@ struct FormatImpl> template static ReturnType execute(const FieldType x, WriteBuffer & wb, const DataTypeDecimal * type, const DateLUTImpl *) { - writeText(x, type->getScale(), wb); + writeText(x, type->getScale(), wb, false); return ReturnType(true); } }; diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 97fd7b77ba6..6a0050b061f 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -901,7 +901,7 @@ inline void writeText(const LocalDateTime & x, WriteBuffer & buf) { writeDateTim inline void writeText(const UUID & x, WriteBuffer & buf) { writeUUIDText(x, buf); } template -void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) +void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr, bool trailing_zeros) { /// If it's big integer, but the number of digits is small, /// use the implementation for smaller integers for more efficient arithmetic. @@ -910,17 +910,17 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) { if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), scale, ostr, trailing_zeros); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), scale, ostr, trailing_zeros); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), scale, ostr, trailing_zeros); return; } } @@ -928,12 +928,12 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) { if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), scale, ostr, trailing_zeros); return; } else if (x <= std::numeric_limits::max()) { - writeDecimalFractional(static_cast(x), scale, ostr); + writeDecimalFractional(static_cast(x), scale, ostr, trailing_zeros); return; } } @@ -957,11 +957,11 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr) } writeChar('.', ostr); - ostr.write(buf, last_nonzero_pos + 1); + ostr.write(buf, trailing_zeros ? scale : last_nonzero_pos + 1); } template -void writeText(Decimal x, UInt32 scale, WriteBuffer & ostr) +void writeText(Decimal x, UInt32 scale, WriteBuffer & ostr, bool trailing_zeros) { T part = DecimalUtils::getWholePart(x, scale); @@ -975,8 +975,8 @@ void writeText(Decimal x, UInt32 scale, WriteBuffer & ostr) if (scale) { part = DecimalUtils::getFractionalPart(x, scale); - if (part) - writeDecimalFractional(part, scale, ostr); + if (part || trailing_zeros) + writeDecimalFractional(part, scale, ostr, trailing_zeros); } } From fabdcf7e71baab00020f1cb1321785179aef0958 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 16 Aug 2021 11:08:53 +0300 Subject: [PATCH 073/220] Update tests --- .../00825_protobuf_format_persons.reference | 104 +++---- ..._index_replicated_zookeeper_long.reference | 16 +- .../0_stateless/00900_long_parquet.reference | 12 +- .../00900_long_parquet_load.reference | 294 +++++++++--------- .../00900_orc_arrays_load.reference | 4 +- .../00900_orc_nullable_arrays_load.reference | 4 +- ...mpression_codecs_replicated_long.reference | 2 +- .../01307_orc_output_format.reference | 12 +- ...761_alter_decimal_zookeeper_long.reference | 14 +- 9 files changed, 231 insertions(+), 231 deletions(-) diff --git a/tests/queries/0_stateless/00825_protobuf_format_persons.reference b/tests/queries/0_stateless/00825_protobuf_format_persons.reference index 711980b3592..897fd9476e9 100644 --- a/tests/queries/0_stateless/00825_protobuf_format_persons.reference +++ b/tests/queries/0_stateless/00825_protobuf_format_persons.reference @@ -1,6 +1,6 @@ -a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] -a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] Schema 00825_protobuf_format_persons:Person @@ -150,9 +150,9 @@ nestiness { Binary representation is as expected Roundtrip: -a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] -a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970680,33.074982] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.97068,33.074982] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] Schema 00825_protobuf_format_persons:AltPerson @@ -264,14 +264,14 @@ nestiness_a_b_c_d: 503 Binary representation is as expected Roundtrip: -a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N 74951234567\0\0 1 2019-01-05 18:45:00 38 capricorn [] [255,0,0] [55.000000,37.000000] 3.140000104904175 214.00 0.1 5.0 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] -c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] [50.000000,-4.000000] 3.141590118408203 \N 0.007 5.0 -20000000000000 [] [] \N [] -a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N 442012345678\0 1 2018-12-30 00:00:00 23 leo [] [250,244,10] [68.000000,33.000000] 3.1415927410125732 100000000000.00 800 -3.0 154400000 ['pound'] [16] 503 [] +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N 74951234567\0\0 1 2019-01-05 18:45:00 38 capricorn [] [255,0,0] [55,37] 3.140000104904175 214 0.1 5 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] [50,-4] 3.141590118408203 \N 0.007 5 -20000000000000 [] [] \N [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N 442012345678\0 1 2018-12-30 00:00:00 23 leo [] [250,244,10] [68,33] 3.1415927410125732 100000000000 800 -3 154400000 ['pound'] [16] 503 [] Schema 00825_protobuf_format_persons:StrPerson Binary representation: -00000000 a7 02 0a 24 61 37 35 32 32 31 35 38 2d 33 64 34 |...$a7522158-3d4| +00000000 a6 02 0a 24 61 37 35 32 32 31 35 38 2d 33 64 34 |...$a7522158-3d4| 00000010 31 2d 34 62 37 37 2d 61 64 36 39 2d 36 63 35 39 |1-4b77-ad69-6c59| 00000020 38 65 65 35 35 63 34 39 12 04 49 76 61 6e 1a 06 |8ee55c49..Ivan..| 00000030 50 65 74 72 6f 76 22 04 6d 61 6c 65 2a 0a 31 39 |Petrov".male*.19| @@ -283,42 +283,42 @@ Binary representation: 00000090 72 73 6a 03 32 35 35 6a 01 30 6a 01 30 72 06 4d |rsj.255j.0j.0r.M| 000000a0 6f 73 63 6f 77 7a 09 35 35 2e 37 35 33 32 31 35 |oscowz.55.753215| 000000b0 7a 09 33 37 2e 36 32 32 35 30 34 82 01 04 33 2e |z.37.622504...3.| -000000c0 31 34 8a 01 06 32 31 34 2e 31 30 92 01 03 30 2e |14...214.10...0.| -000000d0 31 9a 01 03 35 2e 38 a2 01 0b 31 37 30 36 30 30 |1...5.8...170600| -000000e0 30 30 30 30 30 aa 01 2d 0a 05 6d 65 74 65 72 0a |00000..-..meter.| -000000f0 0a 63 65 6e 74 69 6d 65 74 65 72 0a 09 6b 69 6c |.centimeter..kil| -00000100 6f 6d 65 74 65 72 12 01 31 12 04 30 2e 30 31 12 |ometer..1..0.01.| -00000110 04 31 30 30 30 b2 01 11 0a 0f 0a 03 35 30 30 12 |.1000.......500.| -00000120 03 35 30 31 12 03 35 30 32 b4 01 0a 24 63 36 39 |.501..502...$c69| -00000130 34 61 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d |4ad8a-f714-4ea3-| -00000140 39 30 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 |907d-fd54fb25d9b| -00000150 35 12 07 4e 61 74 61 6c 69 61 1a 08 53 6f 6b 6f |5..Natalia..Soko| -00000160 6c 6f 76 61 22 06 66 65 6d 61 6c 65 2a 0a 31 39 |lova".female*.19| -00000170 39 32 2d 30 33 2d 30 38 42 01 30 52 02 32 36 5a |92-03-08B.0R.26Z| -00000180 06 70 69 73 63 65 73 6a 03 31 30 30 6a 03 32 30 |.piscesj.100j.20| -00000190 30 6a 02 35 30 72 08 50 6c 79 6d 6f 75 74 68 7a |0j.50r.Plymouthz| -000001a0 09 35 30 2e 34 30 33 37 32 34 7a 09 2d 34 2e 31 |.50.403724z.-4.1| -000001b0 34 32 31 32 33 82 01 07 33 2e 31 34 31 35 39 92 |42123...3.14159.| -000001c0 01 05 30 2e 30 30 37 9a 01 03 35 2e 34 a2 01 0f |..0.007...5.4...| -000001d0 2d 32 30 30 30 30 30 30 30 30 30 30 30 30 30 84 |-20000000000000.| -000001e0 02 0a 24 61 37 64 61 31 61 61 36 2d 66 34 32 35 |..$a7da1aa6-f425| -000001f0 2d 34 37 38 39 2d 38 39 34 37 2d 62 30 33 34 37 |-4789-8947-b0347| -00000200 38 36 65 64 33 37 34 12 06 56 61 73 69 6c 79 1a |86ed374..Vasily.| -00000210 07 53 69 64 6f 72 6f 76 22 04 6d 61 6c 65 2a 0a |.Sidorov".male*.| -00000220 31 39 39 35 2d 30 37 2d 32 38 3a 0d 2b 34 34 32 |1995-07-28:.+442| -00000230 30 31 32 33 34 35 36 37 38 42 01 31 4a 13 32 30 |012345678B.1J.20| -00000240 31 38 2d 31 32 2d 33 30 20 30 30 3a 30 30 3a 30 |18-12-30 00:00:0| -00000250 30 52 02 32 33 5a 03 6c 65 6f 62 05 53 75 6e 6e |0R.23Z.leob.Sunn| -00000260 79 6a 03 32 35 30 6a 03 32 34 34 6a 02 31 30 72 |yj.250j.244j.10r| -00000270 08 4d 75 72 6d 61 6e 73 6b 7a 09 36 38 2e 39 37 |.Murmanskz.68.97| -00000280 30 36 38 32 7a 09 33 33 2e 30 37 34 39 38 31 82 |0682z.33.074981.| -00000290 01 10 33 2e 31 34 31 35 39 32 36 35 33 35 38 39 |..3.141592653589| -000002a0 37 39 8a 01 0f 31 30 30 30 30 30 30 30 30 30 30 |79...10000000000| -000002b0 30 2e 30 30 92 01 03 38 30 30 9a 01 04 2d 33 2e |0.00...800...-3.| -000002c0 32 a2 01 09 31 35 34 34 30 30 30 30 30 aa 01 0b |2...154400000...| -000002d0 0a 05 70 6f 75 6e 64 12 02 31 36 b2 01 07 0a 05 |..pound..16.....| -000002e0 0a 03 35 30 33 |..503| -000002e5 +000000c0 31 34 8a 01 05 32 31 34 2e 31 92 01 03 30 2e 31 |14...214.1...0.1| +000000d0 9a 01 03 35 2e 38 a2 01 0b 31 37 30 36 30 30 30 |...5.8...1706000| +000000e0 30 30 30 30 aa 01 2d 0a 05 6d 65 74 65 72 0a 0a |0000..-..meter..| +000000f0 63 65 6e 74 69 6d 65 74 65 72 0a 09 6b 69 6c 6f |centimeter..kilo| +00000100 6d 65 74 65 72 12 01 31 12 04 30 2e 30 31 12 04 |meter..1..0.01..| +00000110 31 30 30 30 b2 01 11 0a 0f 0a 03 35 30 30 12 03 |1000.......500..| +00000120 35 30 31 12 03 35 30 32 b4 01 0a 24 63 36 39 34 |501..502...$c694| +00000130 61 64 38 61 2d 66 37 31 34 2d 34 65 61 33 2d 39 |ad8a-f714-4ea3-9| +00000140 30 37 64 2d 66 64 35 34 66 62 32 35 64 39 62 35 |07d-fd54fb25d9b5| +00000150 12 07 4e 61 74 61 6c 69 61 1a 08 53 6f 6b 6f 6c |..Natalia..Sokol| +00000160 6f 76 61 22 06 66 65 6d 61 6c 65 2a 0a 31 39 39 |ova".female*.199| +00000170 32 2d 30 33 2d 30 38 42 01 30 52 02 32 36 5a 06 |2-03-08B.0R.26Z.| +00000180 70 69 73 63 65 73 6a 03 31 30 30 6a 03 32 30 30 |piscesj.100j.200| +00000190 6a 02 35 30 72 08 50 6c 79 6d 6f 75 74 68 7a 09 |j.50r.Plymouthz.| +000001a0 35 30 2e 34 30 33 37 32 34 7a 09 2d 34 2e 31 34 |50.403724z.-4.14| +000001b0 32 31 32 33 82 01 07 33 2e 31 34 31 35 39 92 01 |2123...3.14159..| +000001c0 05 30 2e 30 30 37 9a 01 03 35 2e 34 a2 01 0f 2d |.0.007...5.4...-| +000001d0 32 30 30 30 30 30 30 30 30 30 30 30 30 30 81 02 |20000000000000..| +000001e0 0a 24 61 37 64 61 31 61 61 36 2d 66 34 32 35 2d |.$a7da1aa6-f425-| +000001f0 34 37 38 39 2d 38 39 34 37 2d 62 30 33 34 37 38 |4789-8947-b03478| +00000200 36 65 64 33 37 34 12 06 56 61 73 69 6c 79 1a 07 |6ed374..Vasily..| +00000210 53 69 64 6f 72 6f 76 22 04 6d 61 6c 65 2a 0a 31 |Sidorov".male*.1| +00000220 39 39 35 2d 30 37 2d 32 38 3a 0d 2b 34 34 32 30 |995-07-28:.+4420| +00000230 31 32 33 34 35 36 37 38 42 01 31 4a 13 32 30 31 |12345678B.1J.201| +00000240 38 2d 31 32 2d 33 30 20 30 30 3a 30 30 3a 30 30 |8-12-30 00:00:00| +00000250 52 02 32 33 5a 03 6c 65 6f 62 05 53 75 6e 6e 79 |R.23Z.leob.Sunny| +00000260 6a 03 32 35 30 6a 03 32 34 34 6a 02 31 30 72 08 |j.250j.244j.10r.| +00000270 4d 75 72 6d 61 6e 73 6b 7a 09 36 38 2e 39 37 30 |Murmanskz.68.970| +00000280 36 38 32 7a 09 33 33 2e 30 37 34 39 38 31 82 01 |682z.33.074981..| +00000290 10 33 2e 31 34 31 35 39 32 36 35 33 35 38 39 37 |.3.1415926535897| +000002a0 39 8a 01 0c 31 30 30 30 30 30 30 30 30 30 30 30 |9...100000000000| +000002b0 92 01 03 38 30 30 9a 01 04 2d 33 2e 32 a2 01 09 |...800...-3.2...| +000002c0 31 35 34 34 30 30 30 30 30 aa 01 0b 0a 05 70 6f |154400000.....po| +000002d0 75 6e 64 12 02 31 36 b2 01 07 0a 05 0a 03 35 30 |und..16.......50| +000002e0 33 |3| +000002e1 MESSAGE #1 AT 0x00000002 uuid: "a7522158-3d41-4b77-ad69-6c598ee55c49" @@ -340,7 +340,7 @@ hometown: "Moscow" location: "55.753215" location: "37.622504" pi: "3.14" -lotteryWin: "214.10" +lotteryWin: "214.1" someRatio: "0.1" temperature: "5.8" randomBigNumber: "17060000000" @@ -359,7 +359,7 @@ nestiness_a { e: "502" } } -MESSAGE #2 AT 0x0000012B +MESSAGE #2 AT 0x0000012A uuid: "c694ad8a-f714-4ea3-907d-fd54fb25d9b5" name: "Natalia" surname: "Sokolova" @@ -378,7 +378,7 @@ pi: "3.14159" someRatio: "0.007" temperature: "5.4" randomBigNumber: "-20000000000000" -MESSAGE #3 AT 0x000001E1 +MESSAGE #3 AT 0x000001E0 uuid: "a7da1aa6-f425-4789-8947-b034786ed374" name: "Vasily" surname: "Sidorov" @@ -397,7 +397,7 @@ hometown: "Murmansk" location: "68.970682" location: "33.074981" pi: "3.14159265358979" -lotteryWin: "100000000000.00" +lotteryWin: "100000000000" someRatio: "800" temperature: "-3.2" randomBigNumber: "154400000" @@ -414,9 +414,9 @@ nestiness_a { Binary representation is as expected Roundtrip: -a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 \N +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753215,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 \N \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] -a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 \N +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970682,33.074981] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] Schema 00825_protobuf_format_syntax2:Syntax2Person @@ -564,6 +564,6 @@ Nestiness { Binary representation is as expected Roundtrip: -a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.10 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] +a7522158-3d41-4b77-ad69-6c598ee55c49 Ivan Petrov male 1980-12-29 png +74951234567\0 1 2019-01-05 18:45:00 38 capricorn ['Yesterday','Flowers'] [255,0,0] Moscow [55.753216,37.622504] 3.14 214.1 0.1 5.8 17060000000 ['meter','centimeter','kilometer'] [1,0.01,1000] 500 [501,502] c694ad8a-f714-4ea3-907d-fd54fb25d9b5 Natalia Sokolova female 1992-03-08 jpg \N 0 \N 26 pisces [] [100,200,50] Plymouth [50.403724,-4.142123] 3.14159 \N 0.007 5.4 -20000000000000 [] [] \N [] -a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.970680,33.074982] 3.14159265358979 100000000000.00 800 -3.2 154400000 ['pound'] [16] 503 [] +a7da1aa6-f425-4789-8947-b034786ed374 Vasily Sidorov male 1995-07-28 bmp +442012345678 1 2018-12-30 00:00:00 23 leo ['Sunny'] [250,244,10] Murmansk [68.97068,33.074982] 3.14159265358979 100000000000 800 -3.2 154400000 ['pound'] [16] 503 [] diff --git a/tests/queries/0_stateless/00837_minmax_index_replicated_zookeeper_long.reference b/tests/queries/0_stateless/00837_minmax_index_replicated_zookeeper_long.reference index f5df12ad297..efae13c2a40 100644 --- a/tests/queries/0_stateless/00837_minmax_index_replicated_zookeeper_long.reference +++ b/tests/queries/0_stateless/00837_minmax_index_replicated_zookeeper_long.reference @@ -1,8 +1,8 @@ -0 5 4.7 6.50 cba b 2014-01-04 -1 5 4.7 6.50 cba b 2014-03-11 -12 5 4.7 6.50 cba b 2014-06-11 -13 5 4.7 6.50 cba b 2015-01-01 -0 5 4.7 6.50 cba b 2014-01-04 -1 5 4.7 6.50 cba b 2014-03-11 -12 5 4.7 6.50 cba b 2014-06-11 -13 5 4.7 6.50 cba b 2015-01-01 +0 5 4.7 6.5 cba b 2014-01-04 +1 5 4.7 6.5 cba b 2014-03-11 +12 5 4.7 6.5 cba b 2014-06-11 +13 5 4.7 6.5 cba b 2015-01-01 +0 5 4.7 6.5 cba b 2014-01-04 +1 5 4.7 6.5 cba b 2014-03-11 +12 5 4.7 6.5 cba b 2014-06-11 +13 5 4.7 6.5 cba b 2015-01-01 diff --git a/tests/queries/0_stateless/00900_long_parquet.reference b/tests/queries/0_stateless/00900_long_parquet.reference index d0cb71338af..9ee4fc11a55 100644 --- a/tests/queries/0_stateless/00900_long_parquet.reference +++ b/tests/queries/0_stateless/00900_long_parquet.reference @@ -60,17 +60,17 @@ dest from null: -108 108 -1016 1116 -1032 1132 -1064 1164 -1.032 -1.064 string-0 fixedstring\0\0\0\0 2001-02-03 2002-02-03 04:05:06 127 255 32767 65535 2147483647 4294967295 9223372036854775807 9223372036854775807 -1.032 -1.064 string-2 fixedstring-2\0\0 2004-06-07 2004-02-03 04:05:06 \N \N \N \N \N \N \N \N \N \N \N \N \N \N -1 [1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.20,10.00,4.00] [4.00,10000.10,10000.10] [1000000000.00,90.00,101001.01] -1 [1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.20,10.00,4.00] [4.00,10000.10,10000.10] [1000000000.00,90.00,101001.01] +1 [1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] +1 [1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] 2 [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] 2 [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -1 [1,NULL,2] [NULL,'Some string',NULL] [0.00,NULL,42.42] -1 [1,NULL,2] [NULL,'Some string',NULL] [0.00,NULL,42.42] +1 [1,NULL,2] [NULL,'Some string',NULL] [0,NULL,42.42] +1 [1,NULL,2] [NULL,'Some string',NULL] [0,NULL,42.42] 2 [NULL] [NULL] [NULL] 2 [NULL] [NULL] [NULL] 3 [] [] [] 3 [] [] [] [[[1,2,3],[1,2,3]],[[1,2,3]],[[],[1,2,3]]] [[['Some string','Some string'],[]],[['Some string']],[[]]] [[NULL,1,2],[NULL],[1,2],[]] [['Some string',NULL,'Some string'],[NULL],[]] [[[1,2,3],[1,2,3]],[[1,2,3]],[[],[1,2,3]]] [[['Some string','Some string'],[]],[['Some string']],[[]]] [[NULL,1,2],[NULL],[1,2],[]] [['Some string',NULL,'Some string'],[NULL],[]] -0.1230 0.12312312 0.1231231231230000 0.12312312312312312300000000000000 -0.1230 0.12312312 0.1231231231230000 0.12312312312312312300000000000000 +0.123 0.12312312 0.123123123123 0.123123123123123123 +0.123 0.12312312 0.123123123123 0.123123123123123123 diff --git a/tests/queries/0_stateless/00900_long_parquet_load.reference b/tests/queries/0_stateless/00900_long_parquet_load.reference index f03f56c7125..0fc050891f6 100644 --- a/tests/queries/0_stateless/00900_long_parquet_load.reference +++ b/tests/queries/0_stateless/00900_long_parquet_load.reference @@ -3,8 +3,8 @@ 1 0 1 1 1 10 1.1 10.1 01/01/09 1 1230768060 === Try load data from alltypes_list.parquet [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -[1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.20,10.00,4.00] [4.00,10000.10,10000.10] [1000000000.00,90.00,101001.01] -[1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.20,10.00,4.00] [4.00,10000.10,10000.10] [1000000000.00,90.00,101001.01] +[1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] +[1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] === Try load data from alltypes_plain.parquet 4 1 0 0 0 0 0 0 03/01/09 0 1235865600 5 0 1 1 1 10 1.1 10.1 03/01/09 1 1235865660 @@ -64,30 +64,30 @@ idx10 ['This','is','a','test'] \n === Try load data from byte_array_decimal.parquet -1.00 -2.00 -3.00 -4.00 -5.00 -6.00 -7.00 -8.00 -9.00 -10.00 -11.00 -12.00 -13.00 -14.00 -15.00 -16.00 -17.00 -18.00 -19.00 -20.00 -21.00 -22.00 -23.00 -24.00 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 === Try load data from datapage_v2.snappy.parquet Code: 33. DB::ParsingEx---tion: Error while reading Parquet data: IOError: Not yet implemented: Unsupported encoding.: While executing ParquetBlockInputFormat: data for INSERT was parsed from stdin. (CANNOT_READ_ALL_DATA) @@ -137,135 +137,135 @@ Code: 33. DB::ParsingEx---tion: Error while reading Parquet data: IOError: Not y 1552 1552 === Try load data from fixed_length_decimal.parquet -1.00 -2.00 -3.00 -4.00 -5.00 -6.00 -7.00 -8.00 -9.00 -10.00 -11.00 -12.00 -13.00 -14.00 -15.00 -16.00 -17.00 -18.00 -19.00 -20.00 -21.00 -22.00 -23.00 -24.00 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 === Try load data from fixed_length_decimal_1.parquet -1.00 -2.00 -3.00 -4.00 -5.00 -6.00 -7.00 -8.00 -9.00 -10.00 -11.00 -12.00 -13.00 -14.00 -15.00 -16.00 -17.00 -18.00 -19.00 -20.00 -21.00 -22.00 -23.00 -24.00 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 === Try load data from fixed_length_decimal_legacy.parquet -1.00 -2.00 -3.00 -4.00 -5.00 -6.00 -7.00 -8.00 -9.00 -10.00 -11.00 -12.00 -13.00 -14.00 -15.00 -16.00 -17.00 -18.00 -19.00 -20.00 -21.00 -22.00 -23.00 -24.00 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 === Try load data from hadoop_lz4_compressed.parquet 1593604800 abc 42 1593604800 def 7.7 1593604801 abc 42.125 1593604801 def 7.7 === Try load data from int32_decimal.parquet -1.00 -2.00 -3.00 -4.00 -5.00 -6.00 -7.00 -8.00 -9.00 -10.00 -11.00 -12.00 -13.00 -14.00 -15.00 -16.00 -17.00 -18.00 -19.00 -20.00 -21.00 -22.00 -23.00 -24.00 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 === Try load data from int64_decimal.parquet -1.00 -2.00 -3.00 -4.00 -5.00 -6.00 -7.00 -8.00 -9.00 -10.00 -11.00 -12.00 -13.00 -14.00 -15.00 -16.00 -17.00 -18.00 -19.00 -20.00 -21.00 -22.00 -23.00 -24.00 +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 === Try load data from list_columns.parquet [1,2,3] ['abc','efg','hij'] [NULL,1] [] @@ -325,7 +325,7 @@ Code: 33. DB::ParsingEx---tion: Error while reading Parquet data: IOError: Not y 6 [] [] {} [] (NULL,[],([]),{}) 7 [] [[],[5,6]] {'k1':NULL,'k3':NULL} [] (7,[2,3,NULL],([[],[(NULL,NULL)],[]]),{}) === Try load data from nullable_list.parquet -[1,NULL,2] [NULL,'Some string',NULL] [0.00,NULL,42.42] +[1,NULL,2] [NULL,'Some string',NULL] [0,NULL,42.42] [NULL] [NULL] [NULL] [] [] [] === Try load data from nulls.snappy.parquet diff --git a/tests/queries/0_stateless/00900_orc_arrays_load.reference b/tests/queries/0_stateless/00900_orc_arrays_load.reference index 9b20ef98164..f894669fa0c 100644 --- a/tests/queries/0_stateless/00900_orc_arrays_load.reference +++ b/tests/queries/0_stateless/00900_orc_arrays_load.reference @@ -1,4 +1,4 @@ -[1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.20,10.00,4.00] [4.00,10000.10,10000.10] [1000000000.00,90.00,101001.01] +[1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] -[1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.20,10.00,4.00] [4.00,10000.10,10000.10] [1000000000.00,90.00,101001.01] +[1,-2,3] [1,2,3] [100,-200,300] [100,200,300] [10000000,-20000000,30000000] [10000000,2000000,3000000] [100000000000000,-200000000000,3000000000000] [100000000000000,20000000000000,3000000000000] ['Some string','Some string','Some string'] ['0000','1111','2222'] [42.42,424.2,0.4242] [424242.424242,4242042420.242424,42] ['2000-01-01','2001-01-01','2002-01-01'] ['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'] [0.2,10,4] [4,10000.1,10000.1] [1000000000,90,101001.01] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] [] diff --git a/tests/queries/0_stateless/00900_orc_nullable_arrays_load.reference b/tests/queries/0_stateless/00900_orc_nullable_arrays_load.reference index 62e95652040..44b19f616d0 100644 --- a/tests/queries/0_stateless/00900_orc_nullable_arrays_load.reference +++ b/tests/queries/0_stateless/00900_orc_nullable_arrays_load.reference @@ -1,6 +1,6 @@ -[1,NULL,2] [NULL,'Some string',NULL] [0.00,NULL,42.42] +[1,NULL,2] [NULL,'Some string',NULL] [0,NULL,42.42] [NULL] [NULL] [NULL] [] [] [] -[1,NULL,2] [NULL,'Some string',NULL] [0.00,NULL,42.42] +[1,NULL,2] [NULL,'Some string',NULL] [0,NULL,42.42] [NULL] [NULL] [NULL] [] [] [] diff --git a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated_long.reference b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated_long.reference index 3b7faecbba4..6b7bddf2ac5 100644 --- a/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated_long.reference +++ b/tests/queries/0_stateless/00910_zookeeper_custom_compression_codecs_replicated_long.reference @@ -22,6 +22,6 @@ 9175437371954010821 CREATE TABLE default.compression_codec_multiple_more_types_replicated\n(\n `id` Decimal(38, 13) CODEC(ZSTD(1), LZ4, ZSTD(1), ZSTD(1), Delta(2), Delta(4), Delta(1), LZ4HC(0)),\n `data` FixedString(12) CODEC(ZSTD(1), ZSTD(1), Delta(1), Delta(1), Delta(1), NONE, NONE, NONE, LZ4HC(0)),\n `ddd.age` Array(UInt8) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8)),\n `ddd.Name` Array(String) CODEC(LZ4, LZ4HC(0), NONE, NONE, NONE, ZSTD(1), Delta(8))\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/test_00910/compression_codec_multiple_more_types_replicated\', \'1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 1.5555555555555 hello world! [77] ['John'] -7.1000000000000 xxxxxxxxxxxx [127] ['Henry'] +7.1 xxxxxxxxxxxx [127] ['Henry'] ! 222 diff --git a/tests/queries/0_stateless/01307_orc_output_format.reference b/tests/queries/0_stateless/01307_orc_output_format.reference index da719072eb2..e185c02a3e5 100644 --- a/tests/queries/0_stateless/01307_orc_output_format.reference +++ b/tests/queries/0_stateless/01307_orc_output_format.reference @@ -1,6 +1,6 @@ -255 65535 4294967295 100000000000 -128 -32768 -2147483648 -100000000000 2.02 10000.0000001 String 2020 2021-12-19 2021-12-19 03:00:00 1.0001 1.0000000100 100000.00000000000001000000 1 -4 1234 3244467295 500000000000 -1 -256 -14741221 -7000000000 100.1 14321.032141201 Another string 2000 2024-10-04 2028-04-21 01:20:00 34.1234 123123.1231231230 123123123.12312312312312300000 \N -42 42 42 42 42 42 42 42 42.42 42.42 42 4242 1970-02-12 1970-01-01 03:00:42 42.4200 42.4242424200 424242.42424242424242000000 42 -255 65535 4294967295 100000000000 -128 -32768 -2147483648 -100000000000 2.02 10000.0000001 String 2020 2021-12-19 2021-12-19 03:00:00 1.0001 1.0000000100 100000.00000000000001000000 1 -4 1234 3244467295 500000000000 -1 -256 -14741221 -7000000000 100.1 14321.032141201 Another string 2000 2024-10-04 2028-04-21 01:20:00 34.1234 123123.1231231230 123123123.12312312312312300000 \N -42 42 42 42 42 42 42 42 42.42 42.42 42 4242 1970-02-12 1970-01-01 03:00:42 42.4200 42.4242424200 424242.42424242424242000000 42 +255 65535 4294967295 100000000000 -128 -32768 -2147483648 -100000000000 2.02 10000.0000001 String 2020 2021-12-19 2021-12-19 03:00:00 1.0001 1.00000001 100000.00000000000001 1 +4 1234 3244467295 500000000000 -1 -256 -14741221 -7000000000 100.1 14321.032141201 Another string 2000 2024-10-04 2028-04-21 01:20:00 34.1234 123123.123123123 123123123.123123123123123 \N +42 42 42 42 42 42 42 42 42.42 42.42 42 4242 1970-02-12 1970-01-01 03:00:42 42.42 42.42424242 424242.42424242424242 42 +255 65535 4294967295 100000000000 -128 -32768 -2147483648 -100000000000 2.02 10000.0000001 String 2020 2021-12-19 2021-12-19 03:00:00 1.0001 1.00000001 100000.00000000000001 1 +4 1234 3244467295 500000000000 -1 -256 -14741221 -7000000000 100.1 14321.032141201 Another string 2000 2024-10-04 2028-04-21 01:20:00 34.1234 123123.123123123 123123123.123123123123123 \N +42 42 42 42 42 42 42 42 42.42 42.42 42 4242 1970-02-12 1970-01-01 03:00:42 42.42 42.42424242 424242.42424242424242 42 diff --git a/tests/queries/0_stateless/01761_alter_decimal_zookeeper_long.reference b/tests/queries/0_stateless/01761_alter_decimal_zookeeper_long.reference index ea3f608b6c7..ad5f224bc73 100644 --- a/tests/queries/0_stateless/01761_alter_decimal_zookeeper_long.reference +++ b/tests/queries/0_stateless/01761_alter_decimal_zookeeper_long.reference @@ -1,9 +1,9 @@ -1 5.00000000 -2 6.00000000 +1 5 +2 6 CREATE TABLE default.test_alter_decimal\n(\n `n` UInt64,\n `d` Decimal(18, 8)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/default/01761_alter_decimal_zookeeper\', \'r1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -1 5.00000000 -2 6.00000000 +1 5 +2 6 CREATE TABLE default.test_alter_decimal\n(\n `n` UInt64,\n `d` Decimal(18, 8)\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/default/01761_alter_decimal_zookeeper\', \'r1\')\nORDER BY tuple()\nSETTINGS index_granularity = 8192 -1 5.00000000 -2 6.00000000 -3 7.00000000 +1 5 +2 6 +3 7 From 0b7f6c008ac4e9f2ae5655039139a7761c0f48bc Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Mon, 16 Aug 2021 11:10:59 +0300 Subject: [PATCH 074/220] Update test --- tests/integration/test_mysql_database_engine/test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_mysql_database_engine/test.py b/tests/integration/test_mysql_database_engine/test.py index 8f305fa8463..a093c2a0125 100644 --- a/tests/integration/test_mysql_database_engine/test.py +++ b/tests/integration/test_mysql_database_engine/test.py @@ -232,7 +232,7 @@ uint16_values = [0, 1, 65535] int8_values = [0, 1, -1, 127, -128] uint8_values = [0, 1, 255] # string_values = ["'ClickHouse'", 'NULL'] -string_values = ["'ClickHouse'"] +string_values = ["'ClickHouse'"] decimal_values = [0, 0.123, 0.4, 5.67, 8.91011, 123456789.123, -0.123, -0.4, -5.67, -8.91011, -123456789.123] @@ -319,7 +319,8 @@ def test_mysql_types(started_cluster, case_name, mysql_type, expected_ch_type, m ) clickhouse_query_settings = dict( - mysql_datatypes_support_level=setting_mysql_datatypes_support_level + mysql_datatypes_support_level=setting_mysql_datatypes_support_level, + output_format_decimal_trailing_zeros=1 ) def execute_query(node, query, **kwargs): From c414a3aebf8b819b11dcbf0f541e0d35f9973753 Mon Sep 17 00:00:00 2001 From: jasine Date: Mon, 16 Aug 2021 17:24:51 +0800 Subject: [PATCH 075/220] feat: add docs and tests --- .../functions/type-conversion-functions.md | 144 ++++++++++++++++++ .../01942_dateTimeToSnowflake.reference | 6 + .../0_stateless/01942_dateTimeToSnowflake.sql | 23 +++ .../01942_snowflakeToDateTime.reference | 3 + .../0_stateless/01942_snowflakeToDateTime.sql | 32 ++++ 5 files changed, 208 insertions(+) create mode 100644 tests/queries/0_stateless/01942_dateTimeToSnowflake.reference create mode 100644 tests/queries/0_stateless/01942_dateTimeToSnowflake.sql create mode 100644 tests/queries/0_stateless/01942_snowflakeToDateTime.reference create mode 100644 tests/queries/0_stateless/01942_snowflakeToDateTime.sql diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index efd28def688..5a733f6be23 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1339,3 +1339,147 @@ Result: │ 2,"good" │ └───────────────────────────────────────────┘ ``` + +## snowflakeToDateTime {#snowflakeToDateTime} + +extract time from snowflake id as DateTime format. + +**Syntax** + +``` sql +snowflakeToDateTime(value [, time_zone]) +``` + +**Parameters** + +- `value` — `snowflake id`, Int64 value. +- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- value converted to the `DateTime` data type. + +**Example** + +Query: + +``` sql +SELECT snowflakeToDateTime(CAST('1426860702823350272', 'Int64'), 'UTC'); +``` + +Result: + +``` text + +┌─snowflakeToDateTime(CAST('1426860702823350272', 'Int64'), 'UTC')─┐ +│ 2021-08-15 10:57:56 │ +└──────────────────────────────────────────────────────────────────┘ +``` + +## snowflakeToDateTime64 {#snowflakeToDateTime64} + +extract time from snowflake id as DateTime64 format. + +**Syntax** + +``` sql +snowflakeToDateTime64(value [, time_zone]) +``` + +**Parameters** + +- `value` — `snowflake id`, Int64 value. +- `time_zone` — [Timezone](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-timezone). The function parses `time_string` according to the timezone. Optional. [String](../../sql-reference/data-types/string.md). + +**Returned value** + +- value converted to the `DateTime64` data type. + +**Example** + +Query: + +``` sql +SELECT snowflakeToDateTime64(CAST('1426860802823350272', 'Int64'), 'UTC'); +``` + +Result: + +``` text + +┌─snowflakeToDateTime64(CAST('1426860802823350272', 'Int64'), 'UTC')─┐ +│ 2021-08-15 10:58:19.841 │ +└────────────────────────────────────────────────────────────────────┘ +``` + +## dateTimeToSnowflake {#dateTimeToSnowflake} + +convert DateTime to the first snowflake id at the giving time. + +**Syntax** + +``` sql +dateTimeToSnowflake(value) +``` + +**Parameters** + +- `value` — Date and time. [DateTime](../../sql-reference/data-types/datetime.md). + + +**Returned value** + +- `value` converted to the `Int64` data type as the first snowflake id at that time. + +**Example** + +Query: + +``` sql +SELECT dateTimeToSnowflake(CAST('2021-08-15 18:57:56', 'DateTime')); +``` + +Result: + +``` text + +┌─dateTimeToSnowflake(CAST('2021-08-15 18:57:56', 'DateTime'))─┐ +│ 1426860702823350272 │ +└──────────────────────────────────────────────────────────────┘ +``` + + +## dateTime64ToSnowflake {#dateTime64ToSnowflake} + +convert DateTime64 to the first snowflake id at the giving time. + +**Syntax** + +``` sql +dateTime64ToSnowflake(value) +``` + +**Parameters** + +- `value` — Date and time. [DateTime64](../../sql-reference/data-types/datetime64.md). + + +**Returned value** + +- `value` converted to the `Int64` data type as the first snowflake id at that time. + +**Example** + +Query: + +``` sql +SELECT dateTime64ToSnowflake(CAST('2021-08-15 18:57:56.073', 'DateTime64')); +``` + +Result: + +``` text +┌─dateTime64ToSnowflake(CAST('2021-08-15 18:57:56.073', 'DateTime64'))─┐ +│ 1426860703129534464 │ +└──────────────────────────────────────────────────────────────────────┘ +``` \ No newline at end of file diff --git a/tests/queries/0_stateless/01942_dateTimeToSnowflake.reference b/tests/queries/0_stateless/01942_dateTimeToSnowflake.reference new file mode 100644 index 00000000000..dfca3a10eeb --- /dev/null +++ b/tests/queries/0_stateless/01942_dateTimeToSnowflake.reference @@ -0,0 +1,6 @@ +const column +2021-08-15 18:57:56 1426860702823350272 +2021-08-15 18:57:56.492 1426860704886947840 +non-const column +2021-08-15 18:57:56 1426860702823350272 +2021-08-15 18:57:56.492 1426860704886947840 diff --git a/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql b/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql new file mode 100644 index 00000000000..e5895db7004 --- /dev/null +++ b/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql @@ -0,0 +1,23 @@ +-- Error cases +SELECT dateTimeToSnowflake(); -- {serverError 42} +SELECT dateTime64ToSnowflake(); -- {serverError 42} + +SELECT dateTimeToSnowflake('abc'); -- {serverError 43} +SELECT dateTime64ToSnowflake('abc'); -- {serverError 43} + +SELECT dateTimeToSnowflake('abc', 123); -- {serverError 42} +SELECT dateTime64ToSnowflake('abc', 123); -- {serverError 42} + +SELECT 'const column'; +WITH toDateTime('2021-08-15 18:57:56') AS dt +SELECT dt, dateTimeToSnowflake(dt); + +WITH toDateTime64('2021-08-15 18:57:56.492', 3) AS dt64 +SELECT dt64, dateTime64ToSnowflake(dt64); + +SELECT 'non-const column'; +WITH toDateTime('2021-08-15 18:57:56') AS x +SELECT materialize(x) as dt, dateTimeToSnowflake(dt);; + +WITH toDateTime64('2021-08-15 18:57:56.492', 3) AS x +SELECT materialize(x) as dt64, dateTime64ToSnowflake(dt64); diff --git a/tests/queries/0_stateless/01942_snowflakeToDateTime.reference b/tests/queries/0_stateless/01942_snowflakeToDateTime.reference new file mode 100644 index 00000000000..bed18023f6a --- /dev/null +++ b/tests/queries/0_stateless/01942_snowflakeToDateTime.reference @@ -0,0 +1,3 @@ +const column +UTC 1426860704886947840 2021-08-15 10:57:56 DateTime(\'UTC\') 2021-08-15 10:57:56.492 DateTime64(3, \'UTC\') +Asia/Shanghai 1426860704886947840 2021-08-15 18:57:56 DateTime(\'Asia/Shanghai\') 2021-08-15 18:57:56.492 DateTime64(3, \'Asia/Shanghai\') diff --git a/tests/queries/0_stateless/01942_snowflakeToDateTime.sql b/tests/queries/0_stateless/01942_snowflakeToDateTime.sql new file mode 100644 index 00000000000..f6f171afabf --- /dev/null +++ b/tests/queries/0_stateless/01942_snowflakeToDateTime.sql @@ -0,0 +1,32 @@ +-- -- Error cases +SELECT snowflakeToDateTime(); -- {serverError 42} +SELECT snowflakeToDateTime64(); -- {serverError 42} + +SELECT snowflakeToDateTime('abc'); -- {serverError 43} +SELECT snowflakeToDateTime64('abc'); -- {serverError 43} + +SELECT snowflakeToDateTime('abc', 123); -- {serverError 43} +SELECT snowflakeToDateTime64('abc', 123); -- {serverError 43} + +SELECT 'const column'; +WITH + CAST(1426860704886947840 AS Int64) AS i64, + 'UTC' AS tz +SELECT + tz, + i64, + snowflakeToDateTime(i64, tz) as dt, + toTypeName(dt), + snowflakeToDateTime64(i64, tz) as dt64, + toTypeName(dt64); + +WITH + CAST(1426860704886947840 AS Int64) AS i64, + 'Asia/Shanghai' AS tz +SELECT + tz, + i64, + snowflakeToDateTime(i64, tz) as dt, + toTypeName(dt), + snowflakeToDateTime64(i64, tz) as dt64, + toTypeName(dt64); \ No newline at end of file From 1f21131db680c392e4daeacb47a3ec02b162ef86 Mon Sep 17 00:00:00 2001 From: jasine Date: Mon, 16 Aug 2021 18:52:10 +0800 Subject: [PATCH 076/220] fix: doc and test --- .../functions/type-conversion-functions.md | 18 ++++++++++-------- .../0_stateless/01942_dateTimeToSnowflake.sql | 8 ++++---- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 5a733f6be23..4f1a2d49d23 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1436,16 +1436,17 @@ dateTimeToSnowflake(value) Query: ``` sql -SELECT dateTimeToSnowflake(CAST('2021-08-15 18:57:56', 'DateTime')); +WITH toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt +SELECT dateTimeToSnowflake(dt); ``` Result: ``` text -┌─dateTimeToSnowflake(CAST('2021-08-15 18:57:56', 'DateTime'))─┐ -│ 1426860702823350272 │ -└──────────────────────────────────────────────────────────────┘ +┌─dateTimeToSnowflake(dt)─┐ +│ 1426860702823350272 │ +└─────────────────────────┘ ``` @@ -1473,13 +1474,14 @@ dateTime64ToSnowflake(value) Query: ``` sql -SELECT dateTime64ToSnowflake(CAST('2021-08-15 18:57:56.073', 'DateTime64')); +WITH toDateTime64('2021-08-15 18:57:56.492', 3, 'Asia/Shanghai') AS dt64 +SELECT dateTime64ToSnowflake(dt64); ``` Result: ``` text -┌─dateTime64ToSnowflake(CAST('2021-08-15 18:57:56.073', 'DateTime64'))─┐ -│ 1426860703129534464 │ -└──────────────────────────────────────────────────────────────────────┘ +┌─dateTime64ToSnowflake(dt64)─┐ +│ 1426860704886947840 │ +└─────────────────────────────┘ ``` \ No newline at end of file diff --git a/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql b/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql index e5895db7004..047d8be7be5 100644 --- a/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql +++ b/tests/queries/0_stateless/01942_dateTimeToSnowflake.sql @@ -9,15 +9,15 @@ SELECT dateTimeToSnowflake('abc', 123); -- {serverError 42} SELECT dateTime64ToSnowflake('abc', 123); -- {serverError 42} SELECT 'const column'; -WITH toDateTime('2021-08-15 18:57:56') AS dt +WITH toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS dt SELECT dt, dateTimeToSnowflake(dt); -WITH toDateTime64('2021-08-15 18:57:56.492', 3) AS dt64 +WITH toDateTime64('2021-08-15 18:57:56.492', 3, 'Asia/Shanghai') AS dt64 SELECT dt64, dateTime64ToSnowflake(dt64); SELECT 'non-const column'; -WITH toDateTime('2021-08-15 18:57:56') AS x +WITH toDateTime('2021-08-15 18:57:56', 'Asia/Shanghai') AS x SELECT materialize(x) as dt, dateTimeToSnowflake(dt);; -WITH toDateTime64('2021-08-15 18:57:56.492', 3) AS x +WITH toDateTime64('2021-08-15 18:57:56.492', 3, 'Asia/Shanghai') AS x SELECT materialize(x) as dt64, dateTime64ToSnowflake(dt64); From b162a2b699939c16355b60e1bb607cf74df85865 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 16 Aug 2021 20:09:18 +0800 Subject: [PATCH 077/220] Improve projection analysis. Remove duplicate index analysis and avoid possible invalid limit checks during projection analysis. --- .../QueryPlan/ReadFromMergeTree.cpp | 167 ++++++++++++------ src/Processors/QueryPlan/ReadFromMergeTree.h | 61 ++++++- src/Storages/MergeTree/MergeTreeData.cpp | 31 +++- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 101 ++++------- .../MergeTree/MergeTreeDataSelectExecutor.h | 21 +-- .../MergeTree/StorageFromMergeTreeDataPart.h | 11 +- src/Storages/SelectQueryInfo.h | 6 + 7 files changed, 250 insertions(+), 148 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 4276160f514..1d7a938c6e2 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -40,18 +40,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -struct ReadFromMergeTree::AnalysisResult -{ - RangesInDataParts parts_with_ranges; - MergeTreeDataSelectSamplingData sampling; - IndexStats index_stats; - Names column_names_to_read; - ReadFromMergeTree::ReadType read_type = ReadFromMergeTree::ReadType::Default; - UInt64 selected_rows = 0; - UInt64 selected_marks = 0; - UInt64 selected_parts = 0; -}; - static MergeTreeReaderSettings getMergeTreeReaderSettings(const ContextPtr & context) { const auto & settings = context->getSettingsRef(); @@ -84,7 +72,8 @@ ReadFromMergeTree::ReadFromMergeTree( size_t num_streams_, bool sample_factor_column_queried_, std::shared_ptr max_block_numbers_to_read_, - Poco::Logger * log_) + Poco::Logger * log_, + MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr) : ISourceStep(DataStream{.header = MergeTreeBaseSelectProcessor::transformHeader( metadata_snapshot_->getSampleBlockForColumns(real_column_names_, data_.getVirtuals(), data_.getStorageID()), getPrewhereInfo(query_info_), @@ -116,6 +105,10 @@ ReadFromMergeTree::ReadFromMergeTree( auto type = std::make_shared(); output_stream->header.insert({type->createColumn(), type, "_sample_factor"}); } + + /// If we have analyzed result, reuse it for future planing. + if (analysis_result_ptr) + analyzed_result = analysis_result_ptr->result; } Pipe ReadFromMergeTree::readFromPool( @@ -780,6 +773,33 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( } ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTreeData::DataPartsVector parts) const +{ + return selectRangesToRead( + std::move(parts), + metadata_snapshot_base, + metadata_snapshot, + query_info, + context, + requested_num_streams, + max_block_numbers_to_read, + data, + real_column_names, + sample_factor_column_queried, + log); +} + +ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead( + MergeTreeData::DataPartsVector parts, + const StorageMetadataPtr & metadata_snapshot_base, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + unsigned num_streams, + std::shared_ptr max_block_numbers_to_read, + const MergeTreeData & data, + const Names & real_column_names, + bool sample_factor_column_queried, + Poco::Logger * log) { AnalysisResult result; const auto & settings = context->getSettingsRef(); @@ -808,10 +828,10 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre if (settings.force_primary_key && key_condition.alwaysUnknownOrTrue()) { - throw Exception( - ErrorCodes::INDEX_NOT_USED, - "Primary key ({}) is not used and setting 'force_primary_key' is set.", - fmt::join(primary_key_columns, ", ")); + result.error_msg + = fmt::format("Primary key ({}) is not used and setting 'force_primary_key' is set.", fmt::join(primary_key_columns, ", ")); + result.error_code = ErrorCodes::INDEX_NOT_USED; + return result; } LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); @@ -819,11 +839,30 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre MergeTreeDataSelectExecutor::filterPartsByPartition( parts, part_values, metadata_snapshot_base, data, query_info, context, - max_block_numbers_to_read.get(), log, result.index_stats); + max_block_numbers_to_read.get(), log, result); - result.sampling = MergeTreeDataSelectExecutor::getSampling( - select, metadata_snapshot->getColumns().getAllPhysical(), parts, key_condition, - data, metadata_snapshot, context, sample_factor_column_queried, log); + if (result.error_code) + return result; + + try + { + result.sampling = MergeTreeDataSelectExecutor::getSampling( + select, + metadata_snapshot->getColumns().getAllPhysical(), + parts, + key_condition, + data, + metadata_snapshot, + context, + sample_factor_column_queried, + log); + } + catch (Exception & e) + { + result.error_code = e.code(); + result.error_msg = e.message(); + return result; + } if (result.sampling.read_nothing) return result; @@ -834,18 +873,27 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre size_t parts_before_pk = parts.size(); - result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( - std::move(parts), - metadata_snapshot, - query_info, - context, - key_condition, - reader_settings, - log, - requested_num_streams, - result.index_stats, - true /* use_skip_indexes */, - true /* check_limits */); + try + { + auto reader_settings = getMergeTreeReaderSettings(context); + result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( + std::move(parts), + metadata_snapshot, + query_info, + context, + key_condition, + reader_settings, + log, + num_streams, + result.index_stats, + true /* use_skip_indexes */); + } + catch (Exception & e) + { + result.error_code = e.code(); + result.error_msg = e.message(); + return result; + } size_t sum_marks_pk = total_marks_pk; for (const auto & stat : result.index_stats) @@ -862,23 +910,15 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre sum_marks += part.getMarksCount(); sum_rows += part.getRowsCount(); } - result.selected_parts = result.parts_with_ranges.size(); - result.selected_marks = sum_marks; - result.selected_rows = sum_rows; - LOG_DEBUG( - log, - "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", - parts_before_pk, - total_parts, - result.parts_with_ranges.size(), - sum_marks_pk, - total_marks_pk, - sum_marks, - sum_ranges); - ProfileEvents::increment(ProfileEvents::SelectedParts, result.parts_with_ranges.size()); - ProfileEvents::increment(ProfileEvents::SelectedRanges, sum_ranges); - ProfileEvents::increment(ProfileEvents::SelectedMarks, sum_marks); + result.total_parts = total_parts; + result.parts_before_pk = parts_before_pk; + result.selected_parts = result.parts_with_ranges.size(); + result.selected_ranges = sum_ranges; + result.selected_marks = sum_marks; + result.selected_marks_pk = sum_marks_pk; + result.total_marks_pk = total_marks_pk; + result.selected_rows = sum_rows; const auto & input_order_info = query_info.input_order_info ? query_info.input_order_info @@ -893,7 +933,26 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre void ReadFromMergeTree::initializePipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) { - auto result = selectRangesToRead(prepared_parts); + auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); + + if (result.error_code) + throw Exception(result.error_msg, result.error_code); + + LOG_DEBUG( + log, + "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", + result.parts_before_pk, + result.total_parts, + result.selected_parts, + result.selected_marks_pk, + result.total_marks_pk, + result.selected_marks, + result.selected_ranges); + + ProfileEvents::increment(ProfileEvents::SelectedParts, result.selected_parts); + ProfileEvents::increment(ProfileEvents::SelectedRanges, result.selected_ranges); + ProfileEvents::increment(ProfileEvents::SelectedMarks, result.selected_marks); + auto query_id_holder = MergeTreeDataSelectExecutor::checkLimits(data, result.parts_with_ranges, context); if (result.parts_with_ranges.empty()) @@ -1084,7 +1143,7 @@ static const char * readTypeToString(ReadFromMergeTree::ReadType type) void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const { - auto result = selectRangesToRead(prepared_parts); + auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); std::string prefix(format_settings.offset, format_settings.indent_char); format_settings.out << prefix << "ReadType: " << readTypeToString(result.read_type) << '\n'; @@ -1097,7 +1156,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const { - auto result = selectRangesToRead(prepared_parts); + auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); map.add("Read Type", readTypeToString(result.read_type)); if (!result.index_stats.empty()) { @@ -1108,7 +1167,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const { - auto result = selectRangesToRead(prepared_parts); + auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); auto index_stats = std::move(result.index_stats); std::string prefix(format_settings.offset, format_settings.indent_char); @@ -1160,7 +1219,7 @@ void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const void ReadFromMergeTree::describeIndexes(JSONBuilder::JSONMap & map) const { - auto result = selectRangesToRead(prepared_parts); + auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); auto index_stats = std::move(result.index_stats); if (!index_stats.empty()) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index e83746c3ff0..02c4499ebef 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -9,6 +9,18 @@ using PartitionIdToMaxBlock = std::unordered_map; class Pipe; +struct MergeTreeDataSelectSamplingData +{ + bool use_sampling = false; + bool read_nothing = false; + Float64 used_sample_factor = 1.0; + std::shared_ptr filter_function; + ActionsDAGPtr filter_expression; +}; + +struct MergeTreeDataSelectAnalysisResult; +using MergeTreeDataSelectAnalysisResultPtr = std::shared_ptr; + /// This step is created to read from MergeTree* table. /// For now, it takes a list of parts and creates source from it. class ReadFromMergeTree final : public ISourceStep @@ -54,6 +66,28 @@ public: InReverseOrder, }; + struct AnalysisResult + { + RangesInDataParts parts_with_ranges; + MergeTreeDataSelectSamplingData sampling; + IndexStats index_stats; + Names column_names_to_read; + ReadFromMergeTree::ReadType read_type = ReadFromMergeTree::ReadType::Default; + UInt64 total_parts = 0; + UInt64 parts_before_pk = 0; + UInt64 selected_parts = 0; + UInt64 selected_ranges = 0; + UInt64 selected_marks = 0; + UInt64 selected_marks_pk = 0; + UInt64 total_marks_pk = 0; + UInt64 selected_rows = 0; + bool is_analyzed = false; + + // If error_code is not zero, throw error during initializePipeline. + int error_code = 0; + String error_msg; + }; + ReadFromMergeTree( MergeTreeData::DataPartsVector parts_, Names real_column_names_, @@ -67,7 +101,8 @@ public: size_t num_streams_, bool sample_factor_column_queried_, std::shared_ptr max_block_numbers_to_read_, - Poco::Logger * log_ + Poco::Logger * log_, + MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr ); String getName() const override { return "ReadFromMergeTree"; } @@ -84,6 +119,20 @@ public: UInt64 getSelectedParts() const { return selected_parts; } UInt64 getSelectedRows() const { return selected_rows; } UInt64 getSelectedMarks() const { return selected_marks; } + + static ReadFromMergeTree::AnalysisResult selectRangesToRead( + MergeTreeData::DataPartsVector parts, + const StorageMetadataPtr & metadata_snapshot_base, + const StorageMetadataPtr & metadata_snapshot, + const SelectQueryInfo & query_info, + ContextPtr context, + unsigned num_streams, + std::shared_ptr max_block_numbers_to_read, + const MergeTreeData & data, + const Names & real_column_names, + bool sample_factor_column_queried, + Poco::Logger * log); + private: const MergeTreeReaderSettings reader_settings; @@ -137,8 +186,14 @@ private: const Names & column_names, ActionsDAGPtr & out_projection); - struct AnalysisResult; - AnalysisResult selectRangesToRead(MergeTreeData::DataPartsVector parts) const; + ReadFromMergeTree::AnalysisResult selectRangesToRead(MergeTreeData::DataPartsVector parts) const; + AnalysisResult analyzed_result; +}; + +// For forward declaration. +struct MergeTreeDataSelectAnalysisResult +{ + ReadFromMergeTree::AnalysisResult result; }; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 2892efab12d..bdbb9524b6c 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -51,6 +51,7 @@ #include #include #include +#include #include #include @@ -3940,7 +3941,7 @@ static void selectBestProjection( if (projection_parts.empty()) return; - auto sum_marks = reader.estimateNumMarksToRead( + auto projection_result = reader.estimateNumMarksToRead( projection_parts, candidate.required_columns, metadata_snapshot, @@ -3950,6 +3951,10 @@ static void selectBestProjection( settings.max_threads, max_added_blocks); + if (projection_result.error_code) + return; + + auto sum_marks = projection_result.index_stats.back().num_granules_after; if (normal_parts.empty()) { // All parts are projection parts which allows us to use in_order_optimization. @@ -3958,7 +3963,7 @@ static void selectBestProjection( } else { - sum_marks += reader.estimateNumMarksToRead( + auto normal_result = reader.estimateNumMarksToRead( normal_parts, required_columns, metadata_snapshot, @@ -3967,7 +3972,16 @@ static void selectBestProjection( query_context, settings.max_threads, max_added_blocks); + + if (normal_result.error_code) + return; + + sum_marks += normal_result.index_stats.back().num_granules_after; + candidate.merge_tree_normal_select_result_ptr + = std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(normal_result)}); } + candidate.merge_tree_projection_select_result_ptr + = std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(projection_result)}); // We choose the projection with least sum_marks to read. if (sum_marks < min_sum_marks) @@ -4217,7 +4231,7 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( /// Select the best normal projection if no aggregate projection is available if (!selected_candidate && has_ordinary_projection) { - min_sum_marks = reader.estimateNumMarksToRead( + auto result = reader.estimateNumMarksToRead( parts, analysis_result.required_columns, metadata_snapshot, @@ -4229,7 +4243,7 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( // Add 1 to base sum_marks so that we prefer projections even when they have equal number of marks to read. // NOTE: It is not clear if we need it. E.g. projections do not support skip index for now. - min_sum_marks += 1; + min_sum_marks = result.index_stats.back().num_granules_after + 1; for (auto & candidate : candidates) { @@ -4249,6 +4263,14 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( min_sum_marks); } } + + if (!selected_candidate) + { + // We don't have any good projections, result the MergeTreeDataSelectAnalysisResult for normal scan. + query_info.merge_tree_select_result_ptr = std::make_shared( + MergeTreeDataSelectAnalysisResult{.result = std::move(result)}); + return false; + } } if (!selected_candidate) @@ -4261,7 +4283,6 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( } query_info.projection = std::move(*selected_candidate); - return true; } return false; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index c7eb8200957..b6f50604267 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -145,7 +145,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( context, max_block_size, num_streams, - max_block_numbers_to_read); + max_block_numbers_to_read, + query_info.merge_tree_select_result_ptr); if (plan->isInitialized() && settings.allow_experimental_projection_optimization && settings.force_optimize_projection && !metadata_snapshot->projections.empty()) @@ -190,7 +191,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( context, max_block_size, num_streams, - max_block_numbers_to_read); + max_block_numbers_to_read, + query_info.projection->merge_tree_projection_select_result_ptr); if (plan) { @@ -224,7 +226,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( if (!normal_parts.empty()) { - auto storage_from_base_parts_of_projection = StorageFromMergeTreeDataPart::create(std::move(normal_parts)); + auto storage_from_base_parts_of_projection + = StorageFromMergeTreeDataPart::create(std::move(normal_parts), query_info.projection->merge_tree_normal_select_result_ptr); auto interpreter = InterpreterSelectQuery( query_info.query, context, @@ -666,7 +669,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( const ContextPtr & context, const PartitionIdToMaxBlock * max_block_numbers_to_read, Poco::Logger * log, - ReadFromMergeTree::IndexStats & index_stats) + ReadFromMergeTree::AnalysisResult & result) { const Settings & settings = context->getSettingsRef(); std::optional partition_pruner; @@ -696,7 +699,9 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( } msg += ") nor partition expr is used and setting 'force_index_by_date' is set"; - throw Exception(msg, ErrorCodes::INDEX_NOT_USED); + result.error_msg = msg; + result.error_code = ErrorCodes::INDEX_NOT_USED; + return; } } @@ -724,7 +729,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( max_block_numbers_to_read, part_filter_counters); - index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + result.index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::None, .num_parts_after = part_filter_counters.num_initial_selected_parts, .num_granules_after = part_filter_counters.num_initial_selected_granules}); @@ -732,7 +737,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( if (minmax_idx_condition) { auto description = minmax_idx_condition->getDescription(); - index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + result.index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::MinMax, .condition = std::move(description.condition), .used_keys = std::move(description.used_keys), @@ -744,7 +749,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( if (partition_pruner) { auto description = partition_pruner->getKeyCondition().getDescription(); - index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + result.index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::Partition, .condition = std::move(description.condition), .used_keys = std::move(description.used_keys), @@ -763,8 +768,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd Poco::Logger * log, size_t num_streams, ReadFromMergeTree::IndexStats & index_stats, - bool use_skip_indexes, - bool check_limits) + bool use_skip_indexes) { RangesInDataParts parts_with_ranges(parts.size()); const Settings & settings = context->getSettingsRef(); @@ -892,7 +896,7 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd if (!ranges.ranges.empty()) { - if (check_limits && (limits.max_rows || leaf_limits.max_rows)) + if (limits.max_rows || leaf_limits.max_rows) { /// Fail fast if estimated number of rows to read exceeds the limit auto current_rows_estimate = ranges.getRowsCount(); @@ -1082,7 +1086,7 @@ static void selectColumnNames( } } -size_t MergeTreeDataSelectExecutor::estimateNumMarksToRead( +ReadFromMergeTree::AnalysisResult MergeTreeDataSelectExecutor::estimateNumMarksToRead( MergeTreeData::DataPartsVector parts, const Names & column_names_to_return, const StorageMetadataPtr & metadata_snapshot_base, @@ -1094,7 +1098,11 @@ size_t MergeTreeDataSelectExecutor::estimateNumMarksToRead( { size_t total_parts = parts.size(); if (total_parts == 0) - return 0; + { + ReadFromMergeTree::AnalysisResult result; + result.is_analyzed = true; + return result; + } Names real_column_names; Names virt_column_names; @@ -1104,63 +1112,18 @@ size_t MergeTreeDataSelectExecutor::estimateNumMarksToRead( selectColumnNames(column_names_to_return, data, real_column_names, virt_column_names, sample_factor_column_queried); - auto part_values = filterPartsByVirtualColumns(data, parts, query_info.query, context); - if (part_values && part_values->empty()) - return 0; - - /// If there are only virtual columns in the query, you must request at least one non-virtual one. - if (real_column_names.empty()) - { - NamesAndTypesList available_real_columns = metadata_snapshot->getColumns().getAllPhysical(); - real_column_names.push_back(ExpressionActions::getSmallestColumn(available_real_columns)); - } - - metadata_snapshot->check(real_column_names, data.getVirtuals(), data.getStorageID()); - - const auto & primary_key = metadata_snapshot->getPrimaryKey(); - Names primary_key_columns = primary_key.column_names; - KeyCondition key_condition(query_info, context, primary_key_columns, primary_key.expression); - - if (key_condition.alwaysUnknownOrTrue()) - { - size_t total_marks = 0; - for (const auto & part : parts) - total_marks += part->index_granularity.getMarksCountWithoutFinal(); - - return total_marks; - } - - const auto & select = query_info.query->as(); - ReadFromMergeTree::IndexStats index_stats; - - filterPartsByPartition( - parts, part_values, metadata_snapshot_base, data, query_info, - context, max_block_numbers_to_read.get(), log, index_stats); - - auto sampling = MergeTreeDataSelectExecutor::getSampling( - select, metadata_snapshot->getColumns().getAllPhysical(), parts, key_condition, - data, metadata_snapshot, context, sample_factor_column_queried, log); - - if (sampling.read_nothing) - return 0; - - /// Do not init. It is not used (cause skip index is ignored) - MergeTreeReaderSettings reader_settings; - - auto parts_with_ranges = filterPartsByPrimaryKeyAndSkipIndexes( + return ReadFromMergeTree::selectRangesToRead( std::move(parts), + metadata_snapshot_base, metadata_snapshot, query_info, context, - key_condition, - reader_settings, - log, num_streams, - index_stats, - true /* use_skip_indexes */, - false /* check_limits */); - - return index_stats.back().num_granules_after; + max_block_numbers_to_read, + data, + real_column_names, + sample_factor_column_queried, + log); } QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( @@ -1172,7 +1135,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( ContextPtr context, const UInt64 max_block_size, const unsigned num_streams, - std::shared_ptr max_block_numbers_to_read) const + std::shared_ptr max_block_numbers_to_read, + MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr) const { size_t total_parts = parts.size(); if (total_parts == 0) @@ -1187,7 +1151,7 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( selectColumnNames(column_names_to_return, data, real_column_names, virt_column_names, sample_factor_column_queried); auto read_from_merge_tree = std::make_unique( - parts, + std::move(parts), real_column_names, virt_column_names, data, @@ -1199,7 +1163,8 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( num_streams, sample_factor_column_queried, max_block_numbers_to_read, - log + log, + merge_tree_select_result_ptr ); QueryPlanPtr plan = std::make_unique(); diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index de5ca1f0138..ff21acd7fda 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -13,15 +13,6 @@ namespace DB class KeyCondition; -struct MergeTreeDataSelectSamplingData -{ - bool use_sampling = false; - bool read_nothing = false; - Float64 used_sample_factor = 1.0; - std::shared_ptr filter_function; - ActionsDAGPtr filter_expression; -}; - using PartitionIdToMaxBlock = std::unordered_map; /** Executes SELECT queries on data from the merge tree. @@ -55,12 +46,13 @@ public: ContextPtr context, UInt64 max_block_size, unsigned num_streams, - std::shared_ptr max_block_numbers_to_read = nullptr) const; + std::shared_ptr max_block_numbers_to_read = nullptr, + MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr = nullptr) const; /// Get an estimation for the number of marks we are going to read. /// Reads nothing. Secondary indexes are not used. /// This method is used to select best projection for table. - size_t estimateNumMarksToRead( + ReadFromMergeTree::AnalysisResult estimateNumMarksToRead( MergeTreeData::DataPartsVector parts, const Names & column_names, const StorageMetadataPtr & metadata_snapshot_base, @@ -100,6 +92,8 @@ private: size_t & granules_dropped, Poco::Logger * log); + friend class ReadFromMergeTree; + struct PartFilterCounters { size_t num_initial_selected_parts = 0; @@ -170,7 +164,7 @@ public: const ContextPtr & context, const PartitionIdToMaxBlock * max_block_numbers_to_read, Poco::Logger * log, - ReadFromMergeTree::IndexStats & index_stats); + ReadFromMergeTree::AnalysisResult & result); /// Filter parts using primary key and secondary indexes. /// For every part, select mark ranges to read. @@ -185,8 +179,7 @@ public: Poco::Logger * log, size_t num_streams, ReadFromMergeTree::IndexStats & index_stats, - bool use_skip_indexes, - bool check_limits); + bool use_skip_indexes); /// Create expression for sampling. /// Also, calculate _sample_factor if needed. diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 15beb94404b..26df2e6d658 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -41,7 +41,9 @@ public: query_info, context, max_block_size, - num_streams)); + num_streams, + nullptr, + analysis_result_ptr)); return query_plan.convertToPipe( QueryPlanOptimizationSettings::fromContext(context), BuildQueryPipelineSettings::fromContext(context)); @@ -80,15 +82,16 @@ protected: setInMemoryMetadata(part_->storage.getInMemoryMetadata()); } - StorageFromMergeTreeDataPart(MergeTreeData::DataPartsVector && parts_) - : IStorage(getIDFromParts(parts_)) - , parts(std::move(parts_)) + StorageFromMergeTreeDataPart( + MergeTreeData::DataPartsVector && parts_, MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr_ = nullptr) + : IStorage(getIDFromParts(parts_)), parts(std::move(parts_)), analysis_result_ptr(analysis_result_ptr_) { setInMemoryMetadata(parts.front()->storage.getInMemoryMetadata()); } private: MergeTreeData::DataPartsVector parts; + MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr; static StorageID getIDFromPart(const MergeTreeData::DataPartPtr & part_) { diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 3b3c0fa1258..a4536e1ff58 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -39,6 +39,9 @@ using ReadInOrderOptimizerPtr = std::shared_ptr; class Cluster; using ClusterPtr = std::shared_ptr; +struct MergeTreeDataSelectAnalysisResult; +using MergeTreeDataSelectAnalysisResultPtr = std::shared_ptr; + struct PrewhereInfo { /// Actions which are executed in order to alias columns are used for prewhere actions. @@ -118,6 +121,8 @@ struct ProjectionCandidate ReadInOrderOptimizerPtr order_optimizer; InputOrderInfoPtr input_order_info; ManyExpressionActions group_by_elements_actions; + MergeTreeDataSelectAnalysisResultPtr merge_tree_projection_select_result_ptr; + MergeTreeDataSelectAnalysisResultPtr merge_tree_normal_select_result_ptr; }; /** Query along with some additional data, @@ -158,6 +163,7 @@ struct SelectQueryInfo std::optional projection; bool ignore_projections = false; bool is_projection_query = false; + MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr; }; } From f9a3998351a6d06f8f2e0fc6b0910cdebdd788f5 Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:50:14 +0300 Subject: [PATCH 078/220] Update docs/en/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/en/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 3c3268f89c3..07e80e135bc 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -330,7 +330,7 @@ SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_fami **Example** -Creating a table on multiple servers. After the replica's root directory is lost, the table will attach as read-only as metadata is missing. The last query needs to execute on every replica. +Creating a table on multiple servers. After the replica's metadata in ZooKeeper is lost, the table will attach as read-only as metadata is missing. The last query needs to execute on every replica. ```sql CREATE TABLE test(n UInt32) From a662d2116fd8a0c43df3e67f316a08b8de4627e3 Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:51:02 +0300 Subject: [PATCH 079/220] Update docs/en/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/en/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 07e80e135bc..cf2a99a4c5f 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -348,7 +348,7 @@ SYSTEM RESTORE REPLICA test; Another way: ```sql -RESTORE REPLICA test ON CLUSTER cluster; +SYSTEM RESTORE REPLICA test ON CLUSTER cluster; ``` ### RESTART REPLICAS {#query_language-system-restart-replicas} From 192a9294bd0d9ae5e3cf010b101407cceb7f540d Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:51:35 +0300 Subject: [PATCH 080/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 14ff974ee33..6e4d9279846 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -288,7 +288,7 @@ SYSTEM SYNC REPLICA [db.]replicated_merge_tree_family_table_name ### RESTART REPLICA {#query_language-system-restart-replica} -Реинициализирует состояние сессий Zookeeper для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с Zookeeper (как с эталоном) и при необходимости добавляет задачи в очередь репликации Zookeeper. +Реинициализирует состояние сессий Zookeeper для таблицы семейства `ReplicatedMergeTree`. Сравнивает текущее состояние с состоянием в Zookeeper (как с эталоном) и при необходимости добавляет задачи в очередь репликации в Zookeeper. Инициализация очереди репликации на основе данных ZooKeeper происходит так же, как при `ATTACH TABLE`. Некоторое время таблица будет недоступна для любых операций. ``` sql From 49c54967207f71732437e2101250d44cbb2b558f Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:51:57 +0300 Subject: [PATCH 081/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 6e4d9279846..7cefa7c22e3 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -297,7 +297,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name ### RESTORE REPLICA {#query_language-system-restore-replica} -Восстанавливает реплику, если метаданные Zookeeper потеряны, но сами данные возможно существуют. +Восстанавливает реплику, если метаданные в Zookeeper потеряны, но сами данные возможно существуют. Работает только с таблицами семейства `ReplicatedMergeTree` и только в режиме чтения. From 9091a5a0486f7a842dd2e04ceb4bf5508e91b338 Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:52:55 +0300 Subject: [PATCH 082/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 7cefa7c22e3..5026df16a09 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -299,7 +299,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name Восстанавливает реплику, если метаданные в Zookeeper потеряны, но сами данные возможно существуют. -Работает только с таблицами семейства `ReplicatedMergeTree` и только в режиме чтения. +Работает только с таблицами семейства `ReplicatedMergeTree` и только если таблица находится в readonly-режиме. Запрос можно выполнить из: From 36ac5f9e9411bfcac0054a48d3ab756e7aad3f3b Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:53:37 +0300 Subject: [PATCH 083/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 5026df16a09..846e789f644 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -301,11 +301,11 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name Работает только с таблицами семейства `ReplicatedMergeTree` и только если таблица находится в readonly-режиме. -Запрос можно выполнить из: +Запрос можно выполнить если: - - корневого каталога ZooKeeper `/` с потерянными данными; - - каталога реплики `/replicas` с потерянными данными; - - конкретного пути в каталоге реплики `/replicas/replica_name/` с потерянными данными. + - потерян корневой путь ZooKeeper `/`; + - потерян путь реплик `/replicas`; + - потерян путь конкретной реплики `/replicas/replica_name/`. К реплике прикрепляются локально найденные части, информация о них отправляется в Zookeeper. Если присутствующие в реплике до потери метаданных данные не устарели, они не извлекаются повторно из других реплик. Поэтому восстановление реплики не означает повторную загрузку всех данных по сети. From 8f301ed1f09d768ec4d3f73c101ec1c4ffe944b7 Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:53:53 +0300 Subject: [PATCH 084/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 846e789f644..a682a70e520 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -307,7 +307,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name - потерян путь реплик `/replicas`; - потерян путь конкретной реплики `/replicas/replica_name/`. -К реплике прикрепляются локально найденные части, информация о них отправляется в Zookeeper. +К реплике прикрепляются локально найденные куски, информация о них отправляется в Zookeeper. Если присутствующие в реплике до потери метаданных данные не устарели, они не извлекаются повторно из других реплик. Поэтому восстановление реплики не означает повторную загрузку всех данных по сети. !!! warning "Предупреждение" From 41931b2ed531f0a007fedbc5f4a7102c23d647b6 Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:54:13 +0300 Subject: [PATCH 085/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index a682a70e520..595fb06a2cb 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -308,7 +308,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name - потерян путь конкретной реплики `/replicas/replica_name/`. К реплике прикрепляются локально найденные куски, информация о них отправляется в Zookeeper. -Если присутствующие в реплике до потери метаданных данные не устарели, они не извлекаются повторно из других реплик. Поэтому восстановление реплики не означает повторную загрузку всех данных по сети. +Если присутствующие в реплике до потери метаданных данные не устарели, они не скачиваются повторно с других реплик. Поэтому восстановление реплики не означает повторную загрузку всех данных по сети. !!! warning "Предупреждение" Потерянные данные в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. From 71ce5fbbbdb0cb38c5be76f95d3984667d96cf9b Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:54:34 +0300 Subject: [PATCH 086/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 595fb06a2cb..45b81a05996 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -311,7 +311,7 @@ SYSTEM RESTART REPLICA [db.]replicated_merge_tree_family_table_name Если присутствующие в реплике до потери метаданных данные не устарели, они не скачиваются повторно с других реплик. Поэтому восстановление реплики не означает повторную загрузку всех данных по сети. !!! warning "Предупреждение" - Потерянные данные в любых состояниях перемещаются в папку `detached/`. Части, активные до потери данных (для которых сделан commit), прикрепляются. + Потерянные данные в любых состояниях перемещаются в папку `detached/`. Куски, активные до потери данных (находившиеся в состоянии Committed), прикрепляются. **Синтаксис** From 0c7b114533a356f29c57ceaef791a6fdfa591694 Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:54:49 +0300 Subject: [PATCH 087/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 45b81a05996..2b89c689ba9 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -327,7 +327,7 @@ SYSTEM RESTORE REPLICA [ON CLUSTER cluster_name] [db.]replicated_merge_tree_fami **Пример** -Создание таблицы на нескольких серверах. После потери корневого каталога реплики таблица будет прикреплена только для чтения, так как метаданные отсутствуют. Последний запрос необходимо выполнить на каждой реплике. +Создание таблицы на нескольких серверах. После потери корневого пути реплики таблица будет прикреплена только для чтения, так как метаданные отсутствуют. Последний запрос необходимо выполнить на каждой реплике. ```sql CREATE TABLE test(n UInt32) From 6bb1d7ba86aa940049f37c54ee01deac03b3d2c7 Mon Sep 17 00:00:00 2001 From: pdv-ru <86398979+pdv-ru@users.noreply.github.com> Date: Mon, 16 Aug 2021 15:55:00 +0300 Subject: [PATCH 088/220] Update docs/ru/sql-reference/statements/system.md Co-authored-by: tavplubix --- docs/ru/sql-reference/statements/system.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/sql-reference/statements/system.md b/docs/ru/sql-reference/statements/system.md index 2b89c689ba9..e123f506d46 100644 --- a/docs/ru/sql-reference/statements/system.md +++ b/docs/ru/sql-reference/statements/system.md @@ -345,7 +345,7 @@ SYSTEM RESTORE REPLICA test; Альтернативный способ: ```sql -RESTORE REPLICA test ON CLUSTER cluster; +SYSTEM RESTORE REPLICA test ON CLUSTER cluster; ``` ### RESTART REPLICAS {#query_language-system-restart-replicas} From 3591c3c8f4996dcf801b42c078a52dc7c283e432 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Mon, 16 Aug 2021 13:28:39 +0000 Subject: [PATCH 089/220] correct code according to comments --- programs/client/Client.cpp | 2 +- src/Parsers/ParserInsertQuery.cpp | 16 ++++++++-------- .../getSourceFromFromASTInsertQuery.cpp | 3 +++ tests/queries/0_stateless/02009_from_infile.sh | 4 ++-- 4 files changed, 14 insertions(+), 11 deletions(-) diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index afc75300370..50751de43a4 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -1908,7 +1908,7 @@ private: } catch (Exception & e) { - e.addMessage("data for INSERT was parsed from query"); + e.addMessage("data for INSERT was parsed from file"); throw; } } diff --git a/src/Parsers/ParserInsertQuery.cpp b/src/Parsers/ParserInsertQuery.cpp index 9eb1cbfce02..d597e572437 100644 --- a/src/Parsers/ParserInsertQuery.cpp +++ b/src/Parsers/ParserInsertQuery.cpp @@ -90,17 +90,17 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) Pos before_values = pos; - - /// VALUES or FROM INFILE or FORMAT or SELECT - if (s_values.ignore(pos, expected)) - { - data = pos->begin; - } - else if (s_from_infile.ignore(pos, expected)) + if (s_from_infile.ignore(pos, expected)) { if (!infile_name_p.parse(pos, infile, expected)) return false; } + + /// VALUES or FROM INFILE or FORMAT or SELECT + if (!infile && s_values.ignore(pos, expected)) + { + data = pos->begin; + } else if (s_format.ignore(pos, expected)) { if (!name_p.parse(pos, format, expected)) @@ -146,7 +146,7 @@ bool ParserInsertQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) } - if (format) + if (format && !infile) { Pos last_token = pos; --last_token; diff --git a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp index 8d8a4761657..68a9ec8d95c 100644 --- a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp @@ -35,6 +35,9 @@ Pipe getSourceFromFromASTInsertQuery( if (!ast_insert_query) throw Exception("Logical error: query requires data to insert, but it is not INSERT query", ErrorCodes::LOGICAL_ERROR); + if (ast_insert_query->infile) + throw Exception("Logical error: query has infile and was send directly to server", ErrorCodes::LOGICAL_ERROR); + String format = ast_insert_query->format; if (format.empty()) { diff --git a/tests/queries/0_stateless/02009_from_infile.sh b/tests/queries/0_stateless/02009_from_infile.sh index 6dee54d3963..5cf2bf420a4 100755 --- a/tests/queries/0_stateless/02009_from_infile.sh +++ b/tests/queries/0_stateless/02009_from_infile.sh @@ -9,11 +9,11 @@ set -e [ -e "${CLICKHOUSE_TMP}"/test_infile.gz ] && rm "${CLICKHOUSE_TMP}"/test_infile.gz [ -e "${CLICKHOUSE_TMP}"/test_infile ] && rm "${CLICKHOUSE_TMP}"/test_infile -echo "('Hello')" > "${CLICKHOUSE_TMP}"/test_infile +echo "Hello" > "${CLICKHOUSE_TMP}"/test_infile gzip "${CLICKHOUSE_TMP}"/test_infile ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile (word String) ENGINE=Memory();" -${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz';" +${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV;" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile;" From 90881aab0960ec20aeda6f5950fa4156a34d1f3f Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Mon, 16 Aug 2021 21:23:15 +0800 Subject: [PATCH 090/220] Better code style --- .../QueryPlan/ReadFromMergeTree.cpp | 110 ++++++++++-------- src/Processors/QueryPlan/ReadFromMergeTree.h | 20 ++-- src/Storages/MergeTree/MergeTreeData.cpp | 94 +++++++-------- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 21 ++-- .../MergeTree/MergeTreeDataSelectExecutor.h | 6 +- 5 files changed, 120 insertions(+), 131 deletions(-) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 1d7a938c6e2..dc3e863b841 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -73,7 +73,7 @@ ReadFromMergeTree::ReadFromMergeTree( bool sample_factor_column_queried_, std::shared_ptr max_block_numbers_to_read_, Poco::Logger * log_, - MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr) + MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr_) : ISourceStep(DataStream{.header = MergeTreeBaseSelectProcessor::transformHeader( metadata_snapshot_->getSampleBlockForColumns(real_column_names_, data_.getVirtuals(), data_.getStorageID()), getPrewhereInfo(query_info_), @@ -97,6 +97,7 @@ ReadFromMergeTree::ReadFromMergeTree( , sample_factor_column_queried(sample_factor_column_queried_) , max_block_numbers_to_read(std::move(max_block_numbers_to_read_)) , log(log_) + , analyzed_result_ptr(analyzed_result_ptr_) { if (sample_factor_column_queried) { @@ -105,10 +106,6 @@ ReadFromMergeTree::ReadFromMergeTree( auto type = std::make_shared(); output_stream->header.insert({type->createColumn(), type, "_sample_factor"}); } - - /// If we have analyzed result, reuse it for future planing. - if (analysis_result_ptr) - analyzed_result = analysis_result_ptr->result; } Pipe ReadFromMergeTree::readFromPool( @@ -772,7 +769,7 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( return Pipe::unitePipes(std::move(partition_pipes)); } -ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTreeData::DataPartsVector parts) const +MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead(MergeTreeData::DataPartsVector parts) const { return selectRangesToRead( std::move(parts), @@ -788,7 +785,7 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead(MergeTre log); } -ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead( +MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( MergeTreeData::DataPartsVector parts, const StorageMetadataPtr & metadata_snapshot_base, const StorageMetadataPtr & metadata_snapshot, @@ -808,7 +805,7 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead( auto part_values = MergeTreeDataSelectExecutor::filterPartsByVirtualColumns(data, parts, query_info.query, context); if (part_values && part_values->empty()) - return result; + return std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(result)}); result.column_names_to_read = real_column_names; @@ -828,24 +825,31 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead( if (settings.force_primary_key && key_condition.alwaysUnknownOrTrue()) { - result.error_msg - = fmt::format("Primary key ({}) is not used and setting 'force_primary_key' is set.", fmt::join(primary_key_columns, ", ")); - result.error_code = ErrorCodes::INDEX_NOT_USED; - return result; + return std::make_shared(MergeTreeDataSelectAnalysisResult{ + .result = std::make_exception_ptr(Exception( + ErrorCodes::INDEX_NOT_USED, + "Primary key ({}) is not used and setting 'force_primary_key' is set.", + fmt::join(primary_key_columns, ", ")))}); } LOG_DEBUG(log, "Key condition: {}", key_condition.toString()); const auto & select = query_info.query->as(); - MergeTreeDataSelectExecutor::filterPartsByPartition( - parts, part_values, metadata_snapshot_base, data, query_info, context, - max_block_numbers_to_read.get(), log, result); - - if (result.error_code) - return result; - + size_t total_marks_pk = 0; + size_t parts_before_pk = 0; try { + MergeTreeDataSelectExecutor::filterPartsByPartition( + parts, + part_values, + metadata_snapshot_base, + data, + query_info, + context, + max_block_numbers_to_read.get(), + log, + result.index_stats); + result.sampling = MergeTreeDataSelectExecutor::getSampling( select, metadata_snapshot->getColumns().getAllPhysical(), @@ -856,25 +860,14 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead( context, sample_factor_column_queried, log); - } - catch (Exception & e) - { - result.error_code = e.code(); - result.error_msg = e.message(); - return result; - } - if (result.sampling.read_nothing) - return result; + if (result.sampling.read_nothing) + return std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(result)}); - size_t total_marks_pk = 0; - for (const auto & part : parts) - total_marks_pk += part->index_granularity.getMarksCountWithoutFinal(); + for (const auto & part : parts) + total_marks_pk += part->index_granularity.getMarksCountWithoutFinal(); + parts_before_pk = parts.size(); - size_t parts_before_pk = parts.size(); - - try - { auto reader_settings = getMergeTreeReaderSettings(context); result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( std::move(parts), @@ -888,11 +881,9 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead( result.index_stats, true /* use_skip_indexes */); } - catch (Exception & e) + catch (...) { - result.error_code = e.code(); - result.error_msg = e.message(); - return result; + return std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::current_exception()}); } size_t sum_marks_pk = total_marks_pk; @@ -928,16 +919,21 @@ ReadFromMergeTree::AnalysisResult ReadFromMergeTree::selectRangesToRead( result.read_type = (input_order_info->direction > 0) ? ReadType::InOrder : ReadType::InReverseOrder; - return result; + return std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(result)}); +} + +ReadFromMergeTree::AnalysisResult ReadFromMergeTree::getAnalysisResult() const +{ + auto result_ptr = analyzed_result_ptr ? analyzed_result_ptr : selectRangesToRead(prepared_parts); + if (std::holds_alternative(result_ptr->result)) + std::rethrow_exception(std::move(std::get(result_ptr->result))); + + return std::get(result_ptr->result); } void ReadFromMergeTree::initializePipeline(QueryPipeline & pipeline, const BuildQueryPipelineSettings &) { - auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); - - if (result.error_code) - throw Exception(result.error_msg, result.error_code); - + auto result = getAnalysisResult(); LOG_DEBUG( log, "Selected {}/{} parts by partition key, {} parts by primary key, {}/{} marks by primary key, {} marks to read from {} ranges", @@ -1143,7 +1139,7 @@ static const char * readTypeToString(ReadFromMergeTree::ReadType type) void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const { - auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); std::string prefix(format_settings.offset, format_settings.indent_char); format_settings.out << prefix << "ReadType: " << readTypeToString(result.read_type) << '\n'; @@ -1156,7 +1152,7 @@ void ReadFromMergeTree::describeActions(FormatSettings & format_settings) const void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const { - auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); map.add("Read Type", readTypeToString(result.read_type)); if (!result.index_stats.empty()) { @@ -1167,7 +1163,7 @@ void ReadFromMergeTree::describeActions(JSONBuilder::JSONMap & map) const void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const { - auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); auto index_stats = std::move(result.index_stats); std::string prefix(format_settings.offset, format_settings.indent_char); @@ -1219,7 +1215,7 @@ void ReadFromMergeTree::describeIndexes(FormatSettings & format_settings) const void ReadFromMergeTree::describeIndexes(JSONBuilder::JSONMap & map) const { - auto result = analyzed_result.is_analyzed ? std::move(analyzed_result) : selectRangesToRead(prepared_parts); + auto result = getAnalysisResult(); auto index_stats = std::move(result.index_stats); if (!index_stats.empty()) @@ -1274,4 +1270,20 @@ void ReadFromMergeTree::describeIndexes(JSONBuilder::JSONMap & map) const } } +bool MergeTreeDataSelectAnalysisResult::error() const +{ + return std::holds_alternative(result); +} + +size_t MergeTreeDataSelectAnalysisResult::marks() const +{ + if (std::holds_alternative(result)) + std::rethrow_exception(std::move(std::get(result))); + + const auto & index_stats = std::get(result).index_stats; + if (index_stats.empty()) + return 0; + return index_stats.back().num_granules_after; +} + } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 02c4499ebef..fc06314ee0c 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -81,11 +81,6 @@ public: UInt64 selected_marks_pk = 0; UInt64 total_marks_pk = 0; UInt64 selected_rows = 0; - bool is_analyzed = false; - - // If error_code is not zero, throw error during initializePipeline. - int error_code = 0; - String error_msg; }; ReadFromMergeTree( @@ -102,7 +97,7 @@ public: bool sample_factor_column_queried_, std::shared_ptr max_block_numbers_to_read_, Poco::Logger * log_, - MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr + MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr_ ); String getName() const override { return "ReadFromMergeTree"; } @@ -120,7 +115,7 @@ public: UInt64 getSelectedRows() const { return selected_rows; } UInt64 getSelectedMarks() const { return selected_marks; } - static ReadFromMergeTree::AnalysisResult selectRangesToRead( + static MergeTreeDataSelectAnalysisResultPtr selectRangesToRead( MergeTreeData::DataPartsVector parts, const StorageMetadataPtr & metadata_snapshot_base, const StorageMetadataPtr & metadata_snapshot, @@ -186,14 +181,17 @@ private: const Names & column_names, ActionsDAGPtr & out_projection); - ReadFromMergeTree::AnalysisResult selectRangesToRead(MergeTreeData::DataPartsVector parts) const; - AnalysisResult analyzed_result; + MergeTreeDataSelectAnalysisResultPtr selectRangesToRead(MergeTreeData::DataPartsVector parts) const; + ReadFromMergeTree::AnalysisResult getAnalysisResult() const; + MergeTreeDataSelectAnalysisResultPtr analyzed_result_ptr; }; -// For forward declaration. struct MergeTreeDataSelectAnalysisResult { - ReadFromMergeTree::AnalysisResult result; + std::variant result; + + bool error() const; + size_t marks() const; }; } diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index bdbb9524b6c..743ae00c82f 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3941,7 +3941,7 @@ static void selectBestProjection( if (projection_parts.empty()) return; - auto projection_result = reader.estimateNumMarksToRead( + auto projection_result_ptr = reader.estimateNumMarksToRead( projection_parts, candidate.required_columns, metadata_snapshot, @@ -3951,10 +3951,10 @@ static void selectBestProjection( settings.max_threads, max_added_blocks); - if (projection_result.error_code) + if (projection_result_ptr->error()) return; - auto sum_marks = projection_result.index_stats.back().num_granules_after; + auto sum_marks = projection_result_ptr->marks(); if (normal_parts.empty()) { // All parts are projection parts which allows us to use in_order_optimization. @@ -3963,7 +3963,7 @@ static void selectBestProjection( } else { - auto normal_result = reader.estimateNumMarksToRead( + auto normal_result_ptr = reader.estimateNumMarksToRead( normal_parts, required_columns, metadata_snapshot, @@ -3973,15 +3973,13 @@ static void selectBestProjection( settings.max_threads, max_added_blocks); - if (normal_result.error_code) + if (normal_result_ptr->error()) return; - sum_marks += normal_result.index_stats.back().num_granules_after; - candidate.merge_tree_normal_select_result_ptr - = std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(normal_result)}); + sum_marks += normal_result_ptr->marks(); + candidate.merge_tree_normal_select_result_ptr = normal_result_ptr; } - candidate.merge_tree_projection_select_result_ptr - = std::make_shared(MergeTreeDataSelectAnalysisResult{.result = std::move(projection_result)}); + candidate.merge_tree_projection_select_result_ptr = projection_result_ptr; // We choose the projection with least sum_marks to read. if (sum_marks < min_sum_marks) @@ -4202,10 +4200,25 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( auto parts = getDataPartsVector(); MergeTreeDataSelectExecutor reader(*this); + query_info.merge_tree_select_result_ptr = reader.estimateNumMarksToRead( + parts, + analysis_result.required_columns, + metadata_snapshot, + metadata_snapshot, + query_info, + query_context, + settings.max_threads, + max_added_blocks); + + size_t min_sum_marks = std::numeric_limits::max(); + if (!query_info.merge_tree_select_result_ptr->error()) + { + // Add 1 to base sum_marks so that we prefer projections even when they have equal number of marks to read. + // NOTE: It is not clear if we need it. E.g. projections do not support skip index for now. + min_sum_marks = query_info.merge_tree_select_result_ptr->marks() + 1; + } ProjectionCandidate * selected_candidate = nullptr; - size_t min_sum_marks = std::numeric_limits::max(); - bool has_ordinary_projection = false; /// Favor aggregate projections for (auto & candidate : candidates) { @@ -4224,52 +4237,25 @@ bool MergeTreeData::getQueryProcessingStageWithAggregateProjection( selected_candidate, min_sum_marks); } - else - has_ordinary_projection = true; } - /// Select the best normal projection if no aggregate projection is available - if (!selected_candidate && has_ordinary_projection) + /// Select the best normal projection. + for (auto & candidate : candidates) { - auto result = reader.estimateNumMarksToRead( - parts, - analysis_result.required_columns, - metadata_snapshot, - metadata_snapshot, - query_info, - query_context, - settings.max_threads, - max_added_blocks); - - // Add 1 to base sum_marks so that we prefer projections even when they have equal number of marks to read. - // NOTE: It is not clear if we need it. E.g. projections do not support skip index for now. - min_sum_marks = result.index_stats.back().num_granules_after + 1; - - for (auto & candidate : candidates) + if (candidate.desc->type == ProjectionDescription::Type::Normal) { - if (candidate.desc->type == ProjectionDescription::Type::Normal) - { - selectBestProjection( - reader, - metadata_snapshot, - query_info, - analysis_result.required_columns, - candidate, - query_context, - max_added_blocks, - settings, - parts, - selected_candidate, - min_sum_marks); - } - } - - if (!selected_candidate) - { - // We don't have any good projections, result the MergeTreeDataSelectAnalysisResult for normal scan. - query_info.merge_tree_select_result_ptr = std::make_shared( - MergeTreeDataSelectAnalysisResult{.result = std::move(result)}); - return false; + selectBestProjection( + reader, + metadata_snapshot, + query_info, + analysis_result.required_columns, + candidate, + query_context, + max_added_blocks, + settings, + parts, + selected_candidate, + min_sum_marks); } } diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index b6f50604267..ff0c0657fd9 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -669,7 +669,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( const ContextPtr & context, const PartitionIdToMaxBlock * max_block_numbers_to_read, Poco::Logger * log, - ReadFromMergeTree::AnalysisResult & result) + ReadFromMergeTree::IndexStats & index_stats) { const Settings & settings = context->getSettingsRef(); std::optional partition_pruner; @@ -699,9 +699,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( } msg += ") nor partition expr is used and setting 'force_index_by_date' is set"; - result.error_msg = msg; - result.error_code = ErrorCodes::INDEX_NOT_USED; - return; + throw Exception(msg, ErrorCodes::INDEX_NOT_USED); } } @@ -729,7 +727,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( max_block_numbers_to_read, part_filter_counters); - result.index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::None, .num_parts_after = part_filter_counters.num_initial_selected_parts, .num_granules_after = part_filter_counters.num_initial_selected_granules}); @@ -737,7 +735,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( if (minmax_idx_condition) { auto description = minmax_idx_condition->getDescription(); - result.index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::MinMax, .condition = std::move(description.condition), .used_keys = std::move(description.used_keys), @@ -749,7 +747,7 @@ void MergeTreeDataSelectExecutor::filterPartsByPartition( if (partition_pruner) { auto description = partition_pruner->getKeyCondition().getDescription(); - result.index_stats.emplace_back(ReadFromMergeTree::IndexStat{ + index_stats.emplace_back(ReadFromMergeTree::IndexStat{ .type = ReadFromMergeTree::IndexType::Partition, .condition = std::move(description.condition), .used_keys = std::move(description.used_keys), @@ -1086,7 +1084,7 @@ static void selectColumnNames( } } -ReadFromMergeTree::AnalysisResult MergeTreeDataSelectExecutor::estimateNumMarksToRead( +MergeTreeDataSelectAnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMarksToRead( MergeTreeData::DataPartsVector parts, const Names & column_names_to_return, const StorageMetadataPtr & metadata_snapshot_base, @@ -1098,11 +1096,8 @@ ReadFromMergeTree::AnalysisResult MergeTreeDataSelectExecutor::estimateNumMarksT { size_t total_parts = parts.size(); if (total_parts == 0) - { - ReadFromMergeTree::AnalysisResult result; - result.is_analyzed = true; - return result; - } + return std::make_shared( + MergeTreeDataSelectAnalysisResult{.result = ReadFromMergeTree::AnalysisResult()}); Names real_column_names; Names virt_column_names; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index ff21acd7fda..f8f50723ff0 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -52,7 +52,7 @@ public: /// Get an estimation for the number of marks we are going to read. /// Reads nothing. Secondary indexes are not used. /// This method is used to select best projection for table. - ReadFromMergeTree::AnalysisResult estimateNumMarksToRead( + MergeTreeDataSelectAnalysisResultPtr estimateNumMarksToRead( MergeTreeData::DataPartsVector parts, const Names & column_names, const StorageMetadataPtr & metadata_snapshot_base, @@ -92,8 +92,6 @@ private: size_t & granules_dropped, Poco::Logger * log); - friend class ReadFromMergeTree; - struct PartFilterCounters { size_t num_initial_selected_parts = 0; @@ -164,7 +162,7 @@ public: const ContextPtr & context, const PartitionIdToMaxBlock * max_block_numbers_to_read, Poco::Logger * log, - ReadFromMergeTree::AnalysisResult & result); + ReadFromMergeTree::IndexStats & index_stats); /// Filter parts using primary key and secondary indexes. /// For every part, select mark ranges to read. From 9d0ad10a08694430557788568450664124ec9b15 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 16 Aug 2021 17:24:29 +0300 Subject: [PATCH 091/220] Weaken check a little bit. --- src/Core/Block.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 0f19898ac2f..ddfd62c2efb 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -145,8 +145,8 @@ void Block::insert(size_t position, ColumnWithTypeAndName elem) auto [it, inserted] = index_by_name.emplace(elem.name, position); if (!inserted) - checkColumnStructure(elem, data[it->second], - "(columns with identical name must have identical structure)", false, ErrorCodes::AMBIGUOUS_COLUMN_NAME); + checkColumnStructure(data[it->second], elem, + "(columns with identical name must have identical structure)", true, ErrorCodes::AMBIGUOUS_COLUMN_NAME); data.emplace(data.begin() + position, std::move(elem)); } @@ -159,8 +159,8 @@ void Block::insert(ColumnWithTypeAndName elem) auto [it, inserted] = index_by_name.emplace(elem.name, data.size()); if (!inserted) - checkColumnStructure(elem, data[it->second], - "(columns with identical name must have identical structure)", false, ErrorCodes::AMBIGUOUS_COLUMN_NAME); + checkColumnStructure(data[it->second], elem, + "(columns with identical name must have identical structure)", true, ErrorCodes::AMBIGUOUS_COLUMN_NAME); data.emplace_back(std::move(elem)); } From 43602a838a37e45e6ce9b69d86beee96e840c47f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 16 Aug 2021 18:14:47 +0300 Subject: [PATCH 092/220] Update compare.sh --- docker/test/performance-comparison/compare.sh | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index a6e1ee482d6..e5c9f349ce3 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -628,6 +628,9 @@ cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: clickhouse-local --query " +-- We use decimals specifically to get fixed-point, fixed-width formatting. +set output_format_decimal_trailing_zeros = 1; + create view query_display_names as select * from file('analyze/query-display-names.tsv', TSV, 'test text, query_index int, query_display_name text') @@ -975,6 +978,9 @@ for version in {right,left} do rm -rf data clickhouse-local --query " +-- We use decimals specifically to get fixed-point, fixed-width formatting. +set output_format_decimal_trailing_zeros = 1; + create view query_profiles as with 0 as left, 1 as right select * from file('analyze/query-profiles.tsv', TSV, @@ -1170,6 +1176,9 @@ rm -rf metrics ||: mkdir metrics clickhouse-local --query " +-- We use decimals specifically to get fixed-point, fixed-width formatting. +set output_format_decimal_trailing_zeros = 1; + create view right_async_metric_log as select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes, '$(cat right-async-metric-log.tsv.columns)') From ca65b819d38c644b48d6a0210bebb05e331aebd4 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Mon, 16 Aug 2021 16:09:12 +0000 Subject: [PATCH 093/220] correct error type --- .../Transforms/getSourceFromFromASTInsertQuery.cpp | 3 ++- tests/queries/0_stateless/02009_from_infile.sh | 9 +++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp index 68a9ec8d95c..eb2c1b91cba 100644 --- a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int INVALID_USAGE_OF_INPUT; + extern const int UNKNOWN_TYPE_OF_QUERY; } @@ -36,7 +37,7 @@ Pipe getSourceFromFromASTInsertQuery( throw Exception("Logical error: query requires data to insert, but it is not INSERT query", ErrorCodes::LOGICAL_ERROR); if (ast_insert_query->infile) - throw Exception("Logical error: query has infile and was send directly to server", ErrorCodes::LOGICAL_ERROR); + throw Exception("Query has infile and was send directly to server", ErrorCodes::UNKNOWN_TYPE_OF_QUERY); String format = ast_insert_query->format; if (format.empty()) diff --git a/tests/queries/0_stateless/02009_from_infile.sh b/tests/queries/0_stateless/02009_from_infile.sh index 5cf2bf420a4..4b32ffcd3d5 100755 --- a/tests/queries/0_stateless/02009_from_infile.sh +++ b/tests/queries/0_stateless/02009_from_infile.sh @@ -17,3 +17,12 @@ ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS test_infile;" ${CLICKHOUSE_CLIENT} --query "CREATE TABLE test_infile (word String) ENGINE=Memory();" ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV;" ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile;" + +# if it not fails, select will print information +${CLICKHOUSE_LOCAL} --query "CREATE TABLE test_infile (word String) ENGINE=Memory(); INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV; SELECT * from test_infile;" 2>/dev/null + +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'IF EXISTS test_infile_url' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=CREATE" -d 'TABLE test_infile_url (x String) ENGINE = Memory' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'INSERT INTO test_infile_url FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT x FROM test_infile_url' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'test_infile_url' From e25694e78dbaa3ab45bdbe169f384d535f5c558c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Mon, 16 Aug 2021 19:51:04 +0300 Subject: [PATCH 094/220] Fix test. --- .../0_stateless/01101_literal_column_clash.reference | 3 +++ tests/queries/0_stateless/01101_literal_column_clash.sql | 6 +++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01101_literal_column_clash.reference b/tests/queries/0_stateless/01101_literal_column_clash.reference index 22844815f1e..8f76d98575c 100644 --- a/tests/queries/0_stateless/01101_literal_column_clash.reference +++ b/tests/queries/0_stateless/01101_literal_column_clash.reference @@ -4,4 +4,7 @@ 7 1 xyzabc 2 1 2 0 0 +1 0 0 3 +\N 1 2 \N 0 +\N 1 0 \N 3 2 1 diff --git a/tests/queries/0_stateless/01101_literal_column_clash.sql b/tests/queries/0_stateless/01101_literal_column_clash.sql index ea23f703f9f..b9645e3609e 100644 --- a/tests/queries/0_stateless/01101_literal_column_clash.sql +++ b/tests/queries/0_stateless/01101_literal_column_clash.sql @@ -11,9 +11,9 @@ with 3 as "1" select 1, "1"; -- { serverError 352 } -- https://github.com/ClickHouse/ClickHouse/issues/9953 select 1, * from (select 2 x) a left join (select 1, 3 y) b on y = x; -select 1, * from (select 2 x, 1) a right join (select 3 y) b on y = x; -- { serverError 352 } -select null, isConstant(null), * from (select 2 x) a left join (select null, 3 y) b on y = x; -- { serverError 352 } -select null, isConstant(null), * from (select 2 x, null) a right join (select 3 y) b on y = x; -- { serverError 352 } +select 1, * from (select 2 x, 1) a right join (select 3 y) b on y = x; +select null, isConstant(null), * from (select 2 x) a left join (select null, 3 y) b on y = x; +select null, isConstant(null), * from (select 2 x, null) a right join (select 3 y) b on y = x; -- other cases with joins and constants From 7bbbb19b481a72a9078fa8fe9c120251bea89e28 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Mon, 16 Aug 2021 20:05:50 +0300 Subject: [PATCH 095/220] try to collect some core dumps in perf tests --- docker/test/performance-comparison/entrypoint.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 614debce1c1..fd25a673c85 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -144,8 +144,11 @@ done dmesg -T > dmesg.log +cat /proc/sys/kernel/core_pattern + 7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} \ {right,left}/{performance,scripts} {{right,left}/db,db0}/preprocessed_configs \ - report analyze benchmark metrics + report analyze benchmark metrics \ + ./*.core cp compare.log /output From bc52374f17e14b9c2b3848a9dd74e3b6680d9d51 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Mon, 16 Aug 2021 20:12:12 +0300 Subject: [PATCH 096/220] Translate to Russian MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Выполнил перевод на русский язык. --- .../system-tables/opentelemetry_span_log.md | 2 +- .../operations/system-tables/zookeeper_log.md | 19 +-- .../system-tables/opentelemetry_span_log.md | 2 +- .../operations/system-tables/zookeeper_log.md | 132 ++++++++++++++++++ 4 files changed, 144 insertions(+), 11 deletions(-) create mode 100644 docs/ru/operations/system-tables/zookeeper_log.md diff --git a/docs/en/operations/system-tables/opentelemetry_span_log.md b/docs/en/operations/system-tables/opentelemetry_span_log.md index e45a989742c..9e36eae7a1b 100644 --- a/docs/en/operations/system-tables/opentelemetry_span_log.md +++ b/docs/en/operations/system-tables/opentelemetry_span_log.md @@ -4,7 +4,7 @@ Contains information about [trace spans](https://opentracing.io/docs/overview/sp Columns: -- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md) — ID of the trace for executed query. +- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md)) — ID of the trace for executed query. - `span_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — ID of the `trace span`. diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index 7e24da82e09..25d2d186724 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -1,11 +1,12 @@ # system.zookeeper_log {#system-zookeeper_log} -The table does not exist if ZooKeeper is not configured. - This table contains information about the parameters of the request to the ZooKeeper client and the response from it. For requests, only columns with request parameters are filled in, and the remaining columns are filled with default values (`0` or `NULL`). When the response arrives, the data from the response is added to the other columns. +!!! info "Note" + The table does not exist if ZooKeeper is not configured. + Columns with request parameters: - `type` ([Enum](../../sql-reference/data-types/enum.md)) — Event type in the ZooKeeper client. Can have one of the following values: @@ -15,7 +16,7 @@ Columns with request parameters: - `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the request was completed. - `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the request was completed. - `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the request. -- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — Host port. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the request. - `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — The session ID that the ZooKeeper server sets for each connection. - `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request line and the paired `response`/`finalize` line. - `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The request whether the [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches) has been installed. @@ -25,24 +26,24 @@ Columns with request parameters: - `is_ephemeral` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [ephemeral](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Ephemeral+Nodes). - `is_sequential` ([UInt8](../../sql-reference/data-types/int-uint.md)) — Is the ZooKeeper node being created as an [sequential](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Sequence+Nodes+--+Unique+Naming). - `version` ([Nullable(Int32)](../../sql-reference/data-types/nullable.md)) — The version of the ZooKeeper node that the request expects when executing. This is supported for `CHECK`, `SET`, `REMOVE` requests (is relevant `-1` if the request does not check the version or `NULL` for other requests that do not support version checking). -- `requests_size` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of requests included in the "multi" request (this is a special request that consists of several consecutive ordinary requests and executes them atomically). All requests included in "multi" request will have the same `xid`. -- `request_idx` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of the request included in multi (for multi — `0`, then in order from `1`). +- `requests_size` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of requests included in the multi request (this is a special request that consists of several consecutive ordinary requests and executes them atomically). All requests included in multi request will have the same `xid`. +- `request_idx` ([UInt32](../../sql-reference/data-types/int-uint.md)) — The number of the request included in multi request (for multi request — `0`, then in order from `1`). Columns with request response parameters: -- `zxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — ZooKeeper transaction id. The serial number issued by the ZooKeeper server in response to a successfully executed request (`0` if the request was not executed/returned an error/the client does not know whether the request was executed). +- `zxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — ZooKeeper transaction ID. The serial number issued by the ZooKeeper server in response to a successfully executed request (`0` if the request was not executed/returned an error/the client does not know whether the request was executed). - `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — Error code. Can have one of the following values: - `ZOK` — The response to the request was received. - `ZCONNECTIONLOSS` — The connection was lost. - `ZOPERATIONTIMEOUT` — The request execution timeout has expired. - `ZSESSIONEXPIRED` — The session has expired. - `NULL` — The request is completed. -- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. -- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the "watch" event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. +- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the watch event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. +- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the watch event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. - `path_created` ([String](../../sql-reference/data-types/string.md)) — The path to the created ZooKeeper node (for responses to the `CREATE` request), may differ from the `path` if the node is created as a sequential. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that caused this ZooKeeper node to be created. - `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that last modified this ZooKeeper node. -- `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The transaction id of the change that last modified childern of this ZooKeeper node. +- `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The transaction ID of the change that last modified childern of this ZooKeeper node. - `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — The number of changes to the data of this ZooKeeper node. - `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — The number of changes to the children of this ZooKeeper node. - `stat_dataLength` ([Int32](../../sql-reference/data-types/int-uint.md)) — The length of the data field of this ZooKeeper node. diff --git a/docs/ru/operations/system-tables/opentelemetry_span_log.md b/docs/ru/operations/system-tables/opentelemetry_span_log.md index c421a602300..5c96f22b6c2 100644 --- a/docs/ru/operations/system-tables/opentelemetry_span_log.md +++ b/docs/ru/operations/system-tables/opentelemetry_span_log.md @@ -4,7 +4,7 @@ Столбцы: -- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md) — идентификатор трассировки для выполненного запроса. +- `trace_id` ([UUID](../../sql-reference/data-types/uuid.md)) — идентификатор трассировки для выполненного запроса. - `span_id` ([UInt64](../../sql-reference/data-types/int-uint.md)) — идентификатор `trace span`. diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md new file mode 100644 index 00000000000..0642b8cbad3 --- /dev/null +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -0,0 +1,132 @@ +# system.zookeeper_log {#system-zookeeper_log} + +Эта таблица содержит информацию о параметрах запроса к клиенту ZooKeeper и ответа от него. + +Для запросов заполняются только столбцы с параметрами запроса, а остальные столбцы заполняются значениями по умолчанию (`0` или `NULL`). Когда поступает ответ, данные добавляются в столбцы с параметрами ответа на запрос. + +!!! info "Примечание" + Таблицы не существует, если ZooKeeper не сконфигурирован. + +Столбцы с параметрами запроса: + +- `type` ([Enum](../../sql-reference/data-types/enum.md)) — тип события в клиенте ZooKeeper. Может иметь одно из следующих значений: + - `request` — запрос отправлен. + - `response` — ответ получен. + - `finalize` — соединение разорвано, ответ не получен. +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата завершения выполнения запроса. +- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — дата и время завершения выполнения запроса. +- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого был сделан запрос. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт, с которого был сделан запрос. +- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор сессии, который сервер ZooKeeper устанавливает для каждого соединения. +- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — идентификатор запроса внутри сессии. Обычно это последовательный номер запроса, одинаковый у строки запроса и у парной строки `response`/`finalize`. +- `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — установлен ли запрос [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches). +- `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — тип запроса или ответа на запрос. +- `path` ([String](../../sql-reference/data-types/string.md)) — путь к узлу ZooKeeper, указанный в запросе. Пустая строка, если запрос не требует указания пути. +- `data` ([String](../../sql-reference/data-types/string.md)) — данные, записанные на узле ZooKeeper (для запросов `SET` и `CREATE` — что запрос хотел записать, для ответа на запрос `GET` — что было прочитано), или пустая строка. +- `is_ephemeral` ([UInt8](../../sql-reference/data-types/int-uint.md)) — создается ли узел ZooKeeper как [ephemeral](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Ephemeral+Nodes). +- `is_sequential` ([UInt8](../../sql-reference/data-types/int-uint.md)) — создается ли узел ZooKeeper как [sequential](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#Sequence+Nodes+--+Unique+Naming). +- `version` ([Nullable(Int32)](../../sql-reference/data-types/nullable.md)) — версия узла ZooKeeper, которую запрос ожидает увидеть при выполнении. Поддерживается для запросов `CHECK`, `SET`, `REMOVE` (`-1` — запрос не проверяет версию, `NULL` — для других запросов, которые не поддерживают проверку версии). +- `requests_size` ([UInt32](../../sql-reference/data-types/int-uint.md)) — количество запросов, включенных в мультизапрос (это специальный запрос, который состоит из нескольких последовательных обычных запросов, выполняющихся атомарно). Все запросы, включенные в мультизапрос, имеют одинаковый `xid`. +- `request_idx` ([UInt32](../../sql-reference/data-types/int-uint.md)) — номер запроса, включенного в мультизапрос (`0` — для мультизапроса, далее по порядку с `1`). + +Столбцы с параметрами ответа на запрос: + +- `zxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции в ZooKeeper. Последовательный номер, выданный сервером ZooKeeper в ответе на успешно выполненный запрос (`0` — запрос не был выполнен, возвращена ошибка или клиент ZooKeeper не знает, был ли выполнен запрос). +- `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — код ошибки. Может иметь одно из следующих значений: + - `ZOK` — получен ответ на запрос. + - `ZCONNECTIONLOSS` — соединение разорвано. + - `ZOPERATIONTIMEOUT` — истекло время ожидания выполнения запроса. + - `ZSESSIONEXPIRED` — истекло время сессии. + - `NULL` — выполнен запрос. +- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — тип события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. +- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — статус события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. +- `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как sequential. +- `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — `zxid` изменения, в результате которого был создан узел ZooKeeper. +- `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — `zxid` изменения, которое последним модифицировало узел ZooKeeper. +- `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции изменения, которое последним модифицировало детей узла ZooKeeper. +- `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в данных узла ZooKeeper. +- `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в детях узла ZooKeeper. +- `stat_dataLength` ([Int32](../../sql-reference/data-types/int-uint.md)) — длина поля данных узла ZooKeeper. +- `stat_numChildren` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество детей узла ZooKeeper. +- `children` ([Array(String)](../../sql-reference/data-types/array.md)) — список дочерних узлов ZooKeeper (для ответов на запрос `LIST`). + +**Пример** + +Запрос: + +``` sql +SELECT * FROM system.zookeeper_log WHERE (session_id = '106662742089334927') AND (xid = '10858') FORMAT Vertical; +``` + +Результат: + +``` text +Row 1: +────── +type: Request +event_date: 2021-08-09 +event_time: 2021-08-09 21:38:30.291792 +address: :: +port: 2181 +session_id: 106662742089334927 +xid: 10858 +has_watch: 1 +op_num: List +path: /clickhouse/task_queue/ddl +data: +is_ephemeral: 0 +is_sequential: 0 +version: ᴺᵁᴸᴸ +requests_size: 0 +request_idx: 0 +zxid: 0 +error: ᴺᵁᴸᴸ +watch_type: ᴺᵁᴸᴸ +watch_state: ᴺᵁᴸᴸ +path_created: +stat_czxid: 0 +stat_mzxid: 0 +stat_pzxid: 0 +stat_version: 0 +stat_cversion: 0 +stat_dataLength: 0 +stat_numChildren: 0 +children: [] + +Row 2: +────── +type: Response +event_date: 2021-08-09 +event_time: 2021-08-09 21:38:30.292086 +address: :: +port: 2181 +session_id: 106662742089334927 +xid: 10858 +has_watch: 1 +op_num: List +path: /clickhouse/task_queue/ddl +data: +is_ephemeral: 0 +is_sequential: 0 +version: ᴺᵁᴸᴸ +requests_size: 0 +request_idx: 0 +zxid: 16926267 +error: ZOK +watch_type: ᴺᵁᴸᴸ +watch_state: ᴺᵁᴸᴸ +path_created: +stat_czxid: 16925469 +stat_mzxid: 16925469 +stat_pzxid: 16926179 +stat_version: 0 +stat_cversion: 7 +stat_dataLength: 0 +stat_numChildren: 7 +children: ['query-0000000006','query-0000000005','query-0000000004','query-0000000003','query-0000000002','query-0000000001','query-0000000000'] +``` + +**Смотрите также** + +- [ZooKeeper](../../operations/tips.md#zookeeper) +- [Руководство по ZooKeeper](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html) From 4c1f06258f53cf0fe7d956a9ee584ded80296d86 Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Mon, 16 Aug 2021 17:27:03 +0000 Subject: [PATCH 097/220] correct test --- tests/queries/0_stateless/02009_from_infile.reference | 2 ++ tests/queries/0_stateless/02009_from_infile.sh | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02009_from_infile.reference b/tests/queries/0_stateless/02009_from_infile.reference index e965047ad7c..bfad8971fe4 100644 --- a/tests/queries/0_stateless/02009_from_infile.reference +++ b/tests/queries/0_stateless/02009_from_infile.reference @@ -1 +1,3 @@ Hello +Correct Local +Correct URL diff --git a/tests/queries/0_stateless/02009_from_infile.sh b/tests/queries/0_stateless/02009_from_infile.sh index 4b32ffcd3d5..d50e22d3e6d 100755 --- a/tests/queries/0_stateless/02009_from_infile.sh +++ b/tests/queries/0_stateless/02009_from_infile.sh @@ -19,10 +19,10 @@ ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_ ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile;" # if it not fails, select will print information -${CLICKHOUSE_LOCAL} --query "CREATE TABLE test_infile (word String) ENGINE=Memory(); INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV; SELECT * from test_infile;" 2>/dev/null +${CLICKHOUSE_LOCAL} --query "CREATE TABLE test_infile (word String) ENGINE=Memory(); INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV; SELECT * from test_infile;" 2>&1 | grep -q "UNKNOWN_TYPE_OF_QUERY" && echo "Correct Local" || echo 'Fail' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'IF EXISTS test_infile_url' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=CREATE" -d 'TABLE test_infile_url (x String) ENGINE = Memory' -${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'INSERT INTO test_infile_url FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "INSERT INTO test_infile_url FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV" 2>&1 | grep -q "UNKNOWN_TYPE_OF_QUERY" && echo "Correct URL" || echo 'Fail' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d 'SELECT x FROM test_infile_url' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'test_infile_url' From 9e9fa043ca3c44a431d42e27730d3d5d2553e2e2 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Mon, 16 Aug 2021 21:30:53 +0300 Subject: [PATCH 098/220] minor improvements, add test --- base/daemon/BaseDaemon.cpp | 47 ++++++++++++++ base/daemon/BaseDaemon.h | 7 +++ base/daemon/SentryWriter.cpp | 8 +++ programs/keeper/Keeper.cpp | 2 + programs/server/Server.cpp | 61 +++++-------------- src/Common/getServerUUID.cpp | 12 ++++ src/Common/getServerUUID.h | 5 ++ .../registerFunctionsMiscellaneous.cpp | 2 + src/Functions/serverUUID.cpp | 58 ++++++++++++++++++ .../test_replicated_database/test.py | 9 +++ 10 files changed, 166 insertions(+), 45 deletions(-) create mode 100644 src/Common/getServerUUID.cpp create mode 100644 src/Common/getServerUUID.h create mode 100644 src/Functions/serverUUID.cpp diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 745e020c8bb..060c812590b 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -1059,3 +1060,49 @@ String BaseDaemon::getStoredBinaryHash() const { return stored_binary_hash; } + +void BaseDaemon::loadServerUUID(const fs::path & server_uuid_file, Poco::Logger * log) +{ + /// Write a uuid file containing a unique uuid if the file doesn't already exist during server start. + + if (fs::exists(server_uuid_file)) + { + try + { + DB::UUID uuid; + DB::ReadBufferFromFile in(server_uuid_file); + DB::readUUIDText(uuid, in); + DB::assertEOF(in); + server_uuid = uuid; + return; + } + catch (...) + { + /// As for now it's ok to just overwrite it, because persistency in not essential. + LOG_ERROR(log, "Cannot read server UUID from file {}: {}. Will overwrite it", + server_uuid_file.string(), DB::getCurrentExceptionMessage(true)); + } + } + + try + { + DB::UUID new_uuid = DB::UUIDHelpers::generateV4(); + auto uuid_str = DB::toString(new_uuid); + DB::WriteBufferFromFile out(server_uuid_file); + out.write(uuid_str.data(), uuid_str.size()); + out.sync(); + out.finalize(); + server_uuid = new_uuid; + } + catch (...) + { + throw Poco::Exception( + "Caught Exception " + DB::getCurrentExceptionMessage(true) + " while writing the Server UUID file " + + server_uuid_file.string()); + } +} + +DB::UUID BaseDaemon::getServerUUID() const +{ + return server_uuid; +} diff --git a/base/daemon/BaseDaemon.h b/base/daemon/BaseDaemon.h index 3d47d85a9f5..65c25ae0d57 100644 --- a/base/daemon/BaseDaemon.h +++ b/base/daemon/BaseDaemon.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include +namespace fs = std::filesystem; /// \brief Base class for applications that can run as daemons. /// @@ -124,6 +126,9 @@ public: /// Hash of the binary for integrity checks. String getStoredBinaryHash() const; + void loadServerUUID(const fs::path & server_uuid_file, Poco::Logger * log); + DB::UUID getServerUUID() const; + protected: virtual void logRevision() const; @@ -179,6 +184,8 @@ protected: bool should_setup_watchdog = false; char * argv0 = nullptr; + + DB::UUID server_uuid = DB::UUIDHelpers::Nil; }; diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 3571c64edd6..7578f93f5ed 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #if !defined(ARCADIA_BUILD) # include "Common/config_version.h" @@ -38,6 +39,13 @@ void setExtras() if (!anonymize) sentry_set_extra("server_name", sentry_value_new_string(getFQDNOrHostName().c_str())); + DB::UUID server_uuid = getServerUUID(); + if (server_uuid != DB::UUIDHelpers::Nil) + { + std::string server_uuid_str = DB::toString(server_uuid); + sentry_set_extra("server_uuid", sentry_value_new_string(server_uuid_str.c_str())); + } + sentry_set_tag("version", VERSION_STRING); sentry_set_extra("version_githash", sentry_value_new_string(VERSION_GITHASH)); sentry_set_extra("version_describe", sentry_value_new_string(VERSION_DESCRIBE)); diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 60695cbfeeb..fd225247795 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -326,6 +326,8 @@ int Keeper::main(const std::vector & /*args*/) } } + loadServerUUID(path + "/uuid", log); + const Settings & settings = global_context->getSettingsRef(); GlobalThreadPool::initialize(config().getUInt("max_thread_pool_size", 100)); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index bf1b8e6080d..8685e21ccb4 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -13,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -668,13 +667,14 @@ if (ThreadFuzzer::instance().isEffective()) global_context->setRemoteHostFilter(config()); - std::string path = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); + std::string path_str = getCanonicalPath(config().getString("path", DBMS_DEFAULT_PATH)); + fs::path path = path_str; std::string default_database = config().getString("default_database", "default"); /// Check that the process user id matches the owner of the data. const auto effective_user_id = geteuid(); struct stat statbuf; - if (stat(path.c_str(), &statbuf) == 0 && effective_user_id != statbuf.st_uid) + if (stat(path_str.c_str(), &statbuf) == 0 && effective_user_id != statbuf.st_uid) { const auto effective_user = getUserName(effective_user_id); const auto data_owner = getUserName(statbuf.st_uid); @@ -691,40 +691,11 @@ if (ThreadFuzzer::instance().isEffective()) } } - global_context->setPath(path); + global_context->setPath(path_str); - StatusFile status{path + "status", StatusFile::write_full_info}; + StatusFile status{path / "status", StatusFile::write_full_info}; - - /// Write a uuid file containing a unique uuid if the file doesn't already exist during server start. - { - fs::path server_uuid_file = fs::path(path) / "uuid"; - - if (!fs::exists(server_uuid_file)) - { - try - { - /// Note: Poco::UUIDGenerator().createRandom() uses /dev/random and can be expensive. But since - /// it's only going to be generated once (i.e if the uuid file doesn't exist), it's probably fine. - auto uuid_str = Poco::UUIDGenerator().createRandom().toString(); - WriteBufferFromFile out(server_uuid_file.string()); - out.write(uuid_str.data(), uuid_str.size()); - out.sync(); - out.finalize(); - } - catch (...) - { - throw Poco::Exception( - "Caught Exception " + getCurrentExceptionMessage(false) + " while writing the Server UUID file " - + server_uuid_file.string()); - } - LOG_INFO(log, "Server UUID file {} containing a unique UUID has been written.\n", server_uuid_file.string()); - } - else - { - LOG_INFO(log, "Server UUID file {} already exists, will keep it.\n", server_uuid_file.string()); - } - } + loadServerUUID(path / "uuid", log); /// Try to increase limit on number of open files. { @@ -758,7 +729,7 @@ if (ThreadFuzzer::instance().isEffective()) /// Storage with temporary data for processing of heavy queries. { - std::string tmp_path = config().getString("tmp_path", path + "tmp/"); + std::string tmp_path = config().getString("tmp_path", path / "tmp/"); std::string tmp_policy = config().getString("tmp_policy", ""); const VolumePtr & volume = global_context->setTemporaryStorage(tmp_path, tmp_policy); for (const DiskPtr & disk : volume->getDisks()) @@ -770,7 +741,7 @@ if (ThreadFuzzer::instance().isEffective()) * Examples: do repair of local data; clone all replicated tables from replica. */ { - auto flags_path = fs::path(path) / "flags/"; + auto flags_path = path / "flags/"; fs::create_directories(flags_path); global_context->setFlagsPath(flags_path); } @@ -779,29 +750,29 @@ if (ThreadFuzzer::instance().isEffective()) */ { - std::string user_files_path = config().getString("user_files_path", fs::path(path) / "user_files/"); + std::string user_files_path = config().getString("user_files_path", path / "user_files/"); global_context->setUserFilesPath(user_files_path); fs::create_directories(user_files_path); } { - std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", fs::path(path) / "dictionaries_lib/"); + std::string dictionaries_lib_path = config().getString("dictionaries_lib_path", path / "dictionaries_lib/"); global_context->setDictionariesLibPath(dictionaries_lib_path); fs::create_directories(dictionaries_lib_path); } /// top_level_domains_lists { - const std::string & top_level_domains_path = config().getString("top_level_domains_path", fs::path(path) / "top_level_domains/"); + const std::string & top_level_domains_path = config().getString("top_level_domains_path", path / "top_level_domains/"); TLDListsHolder::getInstance().parseConfig(fs::path(top_level_domains_path) / "", config()); } { - fs::create_directories(fs::path(path) / "data/"); - fs::create_directories(fs::path(path) / "metadata/"); + fs::create_directories(path / "data/"); + fs::create_directories(path / "metadata/"); /// Directory with metadata of tables, which was marked as dropped by Atomic database - fs::create_directories(fs::path(path) / "metadata_dropped/"); + fs::create_directories(path / "metadata_dropped/"); } if (config().has("interserver_http_port") && config().has("interserver_https_port")) @@ -984,7 +955,7 @@ if (ThreadFuzzer::instance().isEffective()) #endif /// Set path for format schema files - fs::path format_schema_path(config().getString("format_schema_path", fs::path(path) / "format_schemas/")); + fs::path format_schema_path(config().getString("format_schema_path", path / "format_schemas/")); global_context->setFormatSchemaPath(format_schema_path); fs::create_directories(format_schema_path); @@ -1120,7 +1091,7 @@ if (ThreadFuzzer::instance().isEffective()) /// system logs may copy global context. global_context->setCurrentDatabaseNameInGlobalContext(default_database); - LOG_INFO(log, "Loading metadata from {}", path); + LOG_INFO(log, "Loading metadata from {}", path_str); try { diff --git a/src/Common/getServerUUID.cpp b/src/Common/getServerUUID.cpp new file mode 100644 index 00000000000..5067bd20c29 --- /dev/null +++ b/src/Common/getServerUUID.cpp @@ -0,0 +1,12 @@ +#include +#include +#include + +DB::UUID getServerUUID() +{ + const auto * daemon = dynamic_cast(&Poco::Util::Application::instance()); + if (daemon) + return daemon->getServerUUID(); + else + return DB::UUIDHelpers::Nil; +} diff --git a/src/Common/getServerUUID.h b/src/Common/getServerUUID.h new file mode 100644 index 00000000000..107dff51f5c --- /dev/null +++ b/src/Common/getServerUUID.h @@ -0,0 +1,5 @@ +#pragma once +#include + +/// Returns persistent UUID of current clickhouse-server or clickhouse-keeper instance. +DB::UUID getServerUUID(); diff --git a/src/Functions/registerFunctionsMiscellaneous.cpp b/src/Functions/registerFunctionsMiscellaneous.cpp index 12c54aeeefd..aed63717155 100644 --- a/src/Functions/registerFunctionsMiscellaneous.cpp +++ b/src/Functions/registerFunctionsMiscellaneous.cpp @@ -78,6 +78,7 @@ void registerFunctionPartitionId(FunctionFactory & factory); void registerFunctionIsIPAddressContainedIn(FunctionFactory &); void registerFunctionQueryID(FunctionFactory & factory); void registerFunctionInitialQueryID(FunctionFactory & factory); +void registerFunctionServerUUID(FunctionFactory &); #if USE_ICU void registerFunctionConvertCharset(FunctionFactory &); @@ -156,6 +157,7 @@ void registerFunctionsMiscellaneous(FunctionFactory & factory) registerFunctionIsIPAddressContainedIn(factory); registerFunctionQueryID(factory); registerFunctionInitialQueryID(factory); + registerFunctionServerUUID(factory); #if USE_ICU registerFunctionConvertCharset(factory); diff --git a/src/Functions/serverUUID.cpp b/src/Functions/serverUUID.cpp new file mode 100644 index 00000000000..5d076ba2a20 --- /dev/null +++ b/src/Functions/serverUUID.cpp @@ -0,0 +1,58 @@ +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ + +class FunctionServerUUID : public IFunction + { + public: + static constexpr auto name = "serverUUID"; + + static FunctionPtr create(ContextPtr context) + { + return std::make_shared(context->isDistributed(), getServerUUID()); + } + + explicit FunctionServerUUID(bool is_distributed_, UUID server_uuid_) + : is_distributed(is_distributed_), server_uuid(server_uuid_) + { + } + + String getName() const override { return name; } + + size_t getNumberOfArguments() const override { return 0; } + + DataTypePtr getReturnTypeImpl(const DataTypes &) const override { return std::make_shared(); } + + bool isDeterministic() const override { return false; } + + bool isDeterministicInScopeOfQuery() const override { return true; } + + bool isSuitableForConstantFolding() const override { return !is_distributed; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName &, const DataTypePtr &, size_t input_rows_count) const override + { + return DataTypeUUID().createColumnConst(input_rows_count, server_uuid); + } + + private: + bool is_distributed; + const UUID server_uuid; + }; + +} + +void registerFunctionServerUUID(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + diff --git a/tests/integration/test_replicated_database/test.py b/tests/integration/test_replicated_database/test.py index 4bcad7de16f..450caafb9ba 100644 --- a/tests/integration/test_replicated_database/test.py +++ b/tests/integration/test_replicated_database/test.py @@ -305,3 +305,12 @@ def test_startup_without_zk(started_cluster): main_node.query("EXCHANGE TABLES startup.rmt AND startup.m") assert main_node.query("SELECT (*,).1 FROM startup.m") == "42\n" + + +def test_server_uuid(started_cluster): + uuid1 = main_node.query("select serverUUID()") + uuid2 = dummy_node.query("select serverUUID()") + assert uuid1 != uuid2 + main_node.restart_clickhouse() + uuid1_after_restart = main_node.query("select serverUUID()") + assert uuid1 == uuid1_after_restart From 1ad4b6ed606b657b5570b26b712992952f4e6d66 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 16 Aug 2021 23:34:39 +0300 Subject: [PATCH 099/220] Fix --- src/Interpreters/InterpreterSelectQuery.cpp | 27 +++++++++++++++++-- ...02004_intersect_except_operators.reference | 7 +++++ .../02004_intersect_except_operators.sql | 5 ++++ 3 files changed, 37 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index edcef191e73..9f4d573735e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -878,16 +879,38 @@ static bool hasWithTotalsInAnySubqueryInFromClause(const ASTSelectQuery & query) { if (const auto * ast_union = query_table->as()) { - /// NOTE: Child of subquery can be ASTSelectWithUnionQuery or ASTSelectQuery, - /// and after normalization, the height of the AST tree is at most 2 for (const auto & elem : ast_union->list_of_selects->children) { + /// After normalization, the height of the AST tree is at most 2. if (const auto * child_union = elem->as()) { for (const auto & child_elem : child_union->list_of_selects->children) if (hasWithTotalsInAnySubqueryInFromClause(child_elem->as())) return true; } + /// After normalization, the height of the AST tree can have any depth, + /// but the number of children at each level is always 2. + else if (const auto * child_intersect_except = elem->as()) + { + std::function traverse_recursively = [&](ASTPtr child_ast) -> bool + { + if (const auto * child = child_ast->as ()) + return hasWithTotalsInAnySubqueryInFromClause(child->as()); + + if (const auto * child = child_ast->as()) + for (const auto & subchild : child->list_of_selects->children) + if (traverse_recursively(subchild)) + return true; + + if (const auto * child = child_ast->as()) + for (const auto & subchild : child->children) + if (traverse_recursively(subchild)) + return true; + return false; + }; + if (traverse_recursively(elem)) + return true; + } else { if (hasWithTotalsInAnySubqueryInFromClause(elem->as())) diff --git a/tests/queries/0_stateless/02004_intersect_except_operators.reference b/tests/queries/0_stateless/02004_intersect_except_operators.reference index 03b881f690b..85559496f2f 100644 --- a/tests/queries/0_stateless/02004_intersect_except_operators.reference +++ b/tests/queries/0_stateless/02004_intersect_except_operators.reference @@ -84,6 +84,13 @@ select count() from (select number from numbers(100) intersect select number fro 30 select count() from (select number from numbers(1000000) intersect select number from numbers(200000, 600000) except select number from numbers(300000, 200000) except select number from numbers(600000, 200000)); 200000 +select count() from (select 1 intersect select 1) limit 100; +1 +select count() from (select 1 except select 2) limit 100; +1 +with (select count() from (select 1 union distinct select 2 except select 1)) as max +select count() from (select 1 union all select max) limit 100; +2 select 1 union all select 1 intersect select 1; 1 1 diff --git a/tests/queries/0_stateless/02004_intersect_except_operators.sql b/tests/queries/0_stateless/02004_intersect_except_operators.sql index 7f08cc0adf2..b95051cba65 100644 --- a/tests/queries/0_stateless/02004_intersect_except_operators.sql +++ b/tests/queries/0_stateless/02004_intersect_except_operators.sql @@ -30,6 +30,11 @@ select count() from (select number from numbers(100) intersect select number fro select count() from (select number from numbers(100) intersect select number from numbers(20, 60) except select number from numbers(30, 20) except select number from numbers(60, 20) union all select number from numbers(100, 10)); select count() from (select number from numbers(1000000) intersect select number from numbers(200000, 600000) except select number from numbers(300000, 200000) except select number from numbers(600000, 200000)); +select count() from (select 1 intersect select 1) limit 100; +select count() from (select 1 except select 2) limit 100; +with (select count() from (select 1 union distinct select 2 except select 1)) as max +select count() from (select 1 union all select max) limit 100; + select 1 union all select 1 intersect select 1; select 1 union all select 1 intersect select 2; select * from (select 1 union all select 2 union all select 3 union all select 4 except select 3 union all select 5) order by 1; From 4ad85ca7452ea0de5967fd2fbacc03a032edf37a Mon Sep 17 00:00:00 2001 From: olgarev Date: Mon, 16 Aug 2021 23:00:38 +0000 Subject: [PATCH 100/220] Initial --- .../mergetree-family/replication.md | 6 +-- .../settings.md | 2 +- .../sql-reference/table-functions/cluster.md | 28 ++++++++++--- .../mergetree-family/replication.md | 6 +-- .../settings.md | 4 +- .../sql-reference/table-functions/cluster.md | 42 ++++++++++++++----- 6 files changed, 64 insertions(+), 24 deletions(-) diff --git a/docs/en/engines/table-engines/mergetree-family/replication.md b/docs/en/engines/table-engines/mergetree-family/replication.md index 4fc30355927..277339f9b47 100644 --- a/docs/en/engines/table-engines/mergetree-family/replication.md +++ b/docs/en/engines/table-engines/mergetree-family/replication.md @@ -137,7 +137,7 @@ CREATE TABLE table_name ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) +SAMPLE BY intHash32(UserID); ```
@@ -150,12 +150,12 @@ CREATE TABLE table_name EventDate DateTime, CounterID UInt32, UserID UInt32 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192); ```
-As the example shows, these parameters can contain substitutions in curly brackets. The substituted values are taken from the «[macros](../../../operations/server-configuration-parameters/settings/#macros) section of the configuration file. +As the example shows, these parameters can contain substitutions in curly brackets. The substituted values are taken from the [macros](../../../operations/server-configuration-parameters/settings.md#macros) section of the configuration file. Example: diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index a620565b71a..ce139312f65 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -486,7 +486,7 @@ Parameter substitutions for replicated tables. Can be omitted if replicated tables are not used. -For more information, see the section “[Creating replicated tables](../../engines/table-engines/mergetree-family/replication.md)”. +For more information, see the section [Creating replicated tables](../../engines/table-engines/mergetree-family/replication.md#creating-replicated-tables). **Example** diff --git a/docs/en/sql-reference/table-functions/cluster.md b/docs/en/sql-reference/table-functions/cluster.md index 2856e66db9b..425015f54b5 100644 --- a/docs/en/sql-reference/table-functions/cluster.md +++ b/docs/en/sql-reference/table-functions/cluster.md @@ -6,12 +6,13 @@ toc_title: cluster # cluster, clusterAllReplicas {#cluster-clusterallreplicas} Allows to access all shards in an existing cluster which configured in `remote_servers` section without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. One replica of each shard is queried. -`clusterAllReplicas` - same as `cluster` but all replicas are queried. Each replica in a cluster is used as separate shard/connection. + +`clusterAllReplicas` function — same as `cluster`, but all replicas are queried. Each replica in a cluster is used as separate shard/connection. !!! note "Note" - All available clusters are listed in the `system.clusters` table. + All available clusters are listed in the [system.clusters](../../operations/system-tables/clusters) table. -Signatures: +**Syntax** ``` sql cluster('cluster_name', db.table[, sharding_key]) @@ -19,10 +20,27 @@ cluster('cluster_name', db, table[, sharding_key]) clusterAllReplicas('cluster_name', db.table[, sharding_key]) clusterAllReplicas('cluster_name', db, table[, sharding_key]) ``` +**Arguments** -`cluster_name` – Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. +- `cluster_name` – Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. +- `db.table` or `db`, `table` - Name of a database and a table. +- `sharding_key` - When insert into cluster function with more than one shard, sharding key needs to be provided. Optional. -`sharding_key` - When insert into cluster function with more than one shard, sharding_key need to be provided. +**Returned value** + +The dataset from clusters. + +**Using Macros** + +Arguments can contain macros — substitutions in curly brackets. The substituted values are taken from the [macros](../../../operations/server-configuration-parameters/settings.md#macros) section of the server configuration file. + +Example: + +```sql +SELECT * FROM cluster('{cluster}', default.example_table); +``` + +**Usage and Recommendations** Using the `cluster` and `clusterAllReplicas` table functions are less efficient than creating a `Distributed` table because in this case, the server connection is re-established for every request. When processing a large number of queries, please always create the `Distributed` table ahead of time, and do not use the `cluster` and `clusterAllReplicas` table functions. diff --git a/docs/ru/engines/table-engines/mergetree-family/replication.md b/docs/ru/engines/table-engines/mergetree-family/replication.md index 6a259ebd3b8..db749ba455e 100644 --- a/docs/ru/engines/table-engines/mergetree-family/replication.md +++ b/docs/ru/engines/table-engines/mergetree-family/replication.md @@ -102,7 +102,7 @@ CREATE TABLE table_name ) ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', ver) PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) -SAMPLE BY intHash32(UserID) +SAMPLE BY intHash32(UserID); ```
@@ -115,12 +115,12 @@ CREATE TABLE table_name EventDate DateTime, CounterID UInt32, UserID UInt32 -) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192) +) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{layer}-{shard}/table_name', '{replica}', EventDate, intHash32(UserID), (CounterID, EventDate, intHash32(UserID), EventTime), 8192); ```
-Как видно в примере, эти параметры могут содержать подстановки в фигурных скобках. Подставляемые значения достаются из конфигурационного файла, из секции «[macros](../../../operations/server-configuration-parameters/settings/#macros)». +Как видно в примере, эти параметры могут содержать подстановки в фигурных скобках. Эти подстановки заменяются на соответствующие значения из конфигурационного файла, из секции [macros](../../../operations/server-configuration-parameters/settings.md#macros). Пример: diff --git a/docs/ru/operations/server-configuration-parameters/settings.md b/docs/ru/operations/server-configuration-parameters/settings.md index a8ae3f7eb3e..98c5748ba41 100644 --- a/docs/ru/operations/server-configuration-parameters/settings.md +++ b/docs/ru/operations/server-configuration-parameters/settings.md @@ -465,9 +465,9 @@ ClickHouse проверяет условия для `min_part_size` и `min_part Подстановки параметров реплицируемых таблиц. -Можно не указывать, если реплицируемых таблицы не используются. +Можно не указывать, если реплицируемые таблицы не используются. -Подробнее смотрите в разделе «[Создание реплицируемых таблиц](../../engines/table-engines/mergetree-family/replication.md)». +Подробнее смотрите в разделе [Создание реплицируемых таблиц](../../engines/table-engines/mergetree-family/replication.md#creating-replicated-tables). **Пример** diff --git a/docs/ru/sql-reference/table-functions/cluster.md b/docs/ru/sql-reference/table-functions/cluster.md index 1a087971afe..8a160e76723 100644 --- a/docs/ru/sql-reference/table-functions/cluster.md +++ b/docs/ru/sql-reference/table-functions/cluster.md @@ -5,22 +5,44 @@ toc_title: cluster # cluster, clusterAllReplicas {#cluster-clusterallreplicas} -Позволяет обратиться ко всем серверам существующего кластера, который присутствует в таблице `system.clusters` и сконфигурирован в секцци `remote_servers` без создания таблицы типа `Distributed`. -`clusterAllReplicas` - работает также как `cluster` но каждая реплика в кластере будет использована как отдельный шард/отдельное соединение. +Позволяет обратиться ко всем шардам существующего кластера, который сконфигурирован в секции `remote_servers` без создания таблицы типа [Distributed](../../engines/table-engines/special/distributed.md). В запросе используется одна реплика каждого шарда. +Функция `clusterAllReplicas` работает также как `cluster`, но каждая реплика в кластере используется как отдельный шард/отдельное соединение. -Сигнатуры: +!!! note "Примечание" + Все доступные кластеры перечислены в таблице [system.clusters](../../operations/system-tables/clusters). + +**Синтаксис** ``` sql -cluster('cluster_name', db.table) -cluster('cluster_name', db, table) -clusterAllReplicas('cluster_name', db.table) -clusterAllReplicas('cluster_name', db, table) +cluster('cluster_name', db.table[, sharding_key]) +cluster('cluster_name', db, table[, sharding_key]) +clusterAllReplicas('cluster_name', db.table[, sharding_key]) +clusterAllReplicas('cluster_name', db, table[, sharding_key]) +``` +**Аргументы** + +- `cluster_name` – имя кластера, который обозначает подмножество адресов и параметров подключения к удаленным и локальным серверам, входящим в кластер. +- `db.table` или `db`, `table` - имя базы данных и таблицы. +- `sharding_key` - ключ шардирования. Указывается, если данные добавляются более чем в один шард кластера. Необязательный аргумент. + +**Возвращаемое значение** + +Набор данных из кластеров. + +**Использование макросов** + +Аргументы могут содержать макросы — подстановки в фигурных скобках. Эти подстановки заменяются на соответствующие значения из конфигурационного файла, из секции [macros](../../../operations/server-configuration-parameters/settings.md#macros). + +Пример: + +```sql +SELECT * FROM cluster('{cluster}', default.example_table); ``` -`cluster_name` – имя кластера, который обязан присутствовать в таблице `system.clusters` и обозначает подмножество адресов и параметров подключения к удаленным и локальным серверам, входящим в кластер. +**Использование и рекомендации** -Использование табличных функций `cluster` и `clusterAllReplicas` менее оптимальное чем создание таблицы типа `Distributed`, поскольку в этом случае соединение с сервером переустанавливается на каждый запрос. При обработке большого количества запросов, всегда создавайте `Distributed` таблицу заранее и не используйте табличные функции `cluster` и `clusterAllReplicas`. +Использование табличных функций `cluster` и `clusterAllReplicas` менее оптимально, чем создание таблицы типа `Distributed`, поскольку в этом случае соединение с сервером переустанавливается на каждый запрос. При обработке большого количества запросов всегда создавайте `Distributed` таблицу заранее и не используйте табличные функции `cluster` и `clusterAllReplicas`. Табличные функции `cluster` and `clusterAllReplicas` могут быть полезны в следующих случаях: @@ -30,7 +52,7 @@ clusterAllReplicas('cluster_name', db, table) Настройки соединения `user`, `password`, `host`, `post`, `compression`, `secure` берутся из секции `` файлов конфигурации. См. подробности в разделе [Distributed](../../engines/table-engines/special/distributed.md) -**See Also** +**См. также** - [skip_unavailable_shards](../../operations/settings/settings.md#settings-skip_unavailable_shards) - [load_balancing](../../operations/settings/settings.md#settings-load_balancing) From 4d044a0657c3cec14702d5b5b0c56d9ed0129644 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 16 Aug 2021 21:26:41 -0300 Subject: [PATCH 101/220] Update graphitemergetree.md --- .../engines/table-engines/mergetree-family/graphitemergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index 30aa10ba38a..1a0ef90c737 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -168,6 +168,6 @@ Fields for `pattern` and `default` sections: ``` !!! warning "Warning" - Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../../sql-reference/statements/optimize/). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). + Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../sql-reference/statements/optimize/optimize.md). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). [Original article](https://clickhouse.tech/docs/en/operations/table_engines/graphitemergetree/) From b1bb073a5f4c268c4d870d8f62495248e7d9d99e Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Mon, 16 Aug 2021 21:27:07 -0300 Subject: [PATCH 102/220] Update graphitemergetree.md --- .../engines/table-engines/mergetree-family/graphitemergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md index 599e08bc7c3..08546f960aa 100644 --- a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md @@ -171,4 +171,4 @@ default !!! warning "Внимание" - Прореживание данных производится во время слияний. Обычно для старых партций слияния не запускаются, поэтому для прореживания надо иницировать незапланированное слияние используя [optimize](../../../../sql-reference/statements/optimize/). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). + Прореживание данных производится во время слияний. Обычно для старых партций слияния не запускаются, поэтому для прореживания надо иницировать незапланированное слияние используя [optimize](../../../sql-reference/statements/optimize/optimize.md). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). From 9b44146b9fe330539328c05309871dd2f9880307 Mon Sep 17 00:00:00 2001 From: olgarev Date: Tue, 17 Aug 2021 01:54:48 +0000 Subject: [PATCH 103/220] Links fixed --- clickhouse-fork | 1 + docs/en/sql-reference/table-functions/cluster.md | 4 ++-- docs/ru/sql-reference/table-functions/cluster.md | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) create mode 160000 clickhouse-fork diff --git a/clickhouse-fork b/clickhouse-fork new file mode 160000 index 00000000000..157bca84f41 --- /dev/null +++ b/clickhouse-fork @@ -0,0 +1 @@ +Subproject commit 157bca84f412a0cf25497908ed19bf5a66f0aaec diff --git a/docs/en/sql-reference/table-functions/cluster.md b/docs/en/sql-reference/table-functions/cluster.md index 425015f54b5..d392cc80dc0 100644 --- a/docs/en/sql-reference/table-functions/cluster.md +++ b/docs/en/sql-reference/table-functions/cluster.md @@ -10,7 +10,7 @@ Allows to access all shards in an existing cluster which configured in `remote_s `clusterAllReplicas` function — same as `cluster`, but all replicas are queried. Each replica in a cluster is used as separate shard/connection. !!! note "Note" - All available clusters are listed in the [system.clusters](../../operations/system-tables/clusters) table. + All available clusters are listed in the [system.clusters](../../operations/system-tables/clusters.md) table. **Syntax** @@ -32,7 +32,7 @@ The dataset from clusters. **Using Macros** -Arguments can contain macros — substitutions in curly brackets. The substituted values are taken from the [macros](../../../operations/server-configuration-parameters/settings.md#macros) section of the server configuration file. +`cluster_name` can contain macros — substitution in curly brackets. The substituted value is taken from the [macros](../../operations/server-configuration-parameters/settings.md#macros) section of the server configuration file. Example: diff --git a/docs/ru/sql-reference/table-functions/cluster.md b/docs/ru/sql-reference/table-functions/cluster.md index 8a160e76723..e961e54dda4 100644 --- a/docs/ru/sql-reference/table-functions/cluster.md +++ b/docs/ru/sql-reference/table-functions/cluster.md @@ -10,7 +10,7 @@ toc_title: cluster Функция `clusterAllReplicas` работает также как `cluster`, но каждая реплика в кластере используется как отдельный шард/отдельное соединение. !!! note "Примечание" - Все доступные кластеры перечислены в таблице [system.clusters](../../operations/system-tables/clusters). + Все доступные кластеры перечислены в таблице [system.clusters](../../operations/system-tables/clusters.md). **Синтаксис** @@ -32,7 +32,7 @@ clusterAllReplicas('cluster_name', db, table[, sharding_key]) **Использование макросов** -Аргументы могут содержать макросы — подстановки в фигурных скобках. Эти подстановки заменяются на соответствующие значения из конфигурационного файла, из секции [macros](../../../operations/server-configuration-parameters/settings.md#macros). +`cluster_name` может содержать макрос — подстановку в фигурных скобках. Эта подстановка заменяется на соответствующее значение из конфигурационного файла, из секции [macros](../../operations/server-configuration-parameters/settings.md#macros). Пример: From ceb14a0c53f6f6fd904208b06f61910c4cf95d5d Mon Sep 17 00:00:00 2001 From: olgarev Date: Tue, 17 Aug 2021 02:07:28 +0000 Subject: [PATCH 104/220] Delete wrong folder --- clickhouse-fork | 1 - 1 file changed, 1 deletion(-) delete mode 160000 clickhouse-fork diff --git a/clickhouse-fork b/clickhouse-fork deleted file mode 160000 index 157bca84f41..00000000000 --- a/clickhouse-fork +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 157bca84f412a0cf25497908ed19bf5a66f0aaec From ed4845e8f2b28b2e0bae3ce64212bfd62bbed64e Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Tue, 17 Aug 2021 00:13:51 -0300 Subject: [PATCH 105/220] Update graphitemergetree.md --- .../engines/table-engines/mergetree-family/graphitemergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md index 08546f960aa..fab796b8413 100644 --- a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md @@ -171,4 +171,4 @@ default !!! warning "Внимание" - Прореживание данных производится во время слияний. Обычно для старых партций слияния не запускаются, поэтому для прореживания надо иницировать незапланированное слияние используя [optimize](../../../sql-reference/statements/optimize/optimize.md). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). + Прореживание данных производится во время слияний. Обычно для старых партций слияния не запускаются, поэтому для прореживания надо иницировать незапланированное слияние используя [optimize](../../../sql-reference/statements/optimize.md). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). From 344016ee0eb3826177dcaaa1ac77fe122902da95 Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Tue, 17 Aug 2021 00:15:07 -0300 Subject: [PATCH 106/220] Update graphitemergetree.md --- .../engines/table-engines/mergetree-family/graphitemergetree.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index 1a0ef90c737..33707709fdf 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -168,6 +168,6 @@ Fields for `pattern` and `default` sections: ``` !!! warning "Warning" - Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../sql-reference/statements/optimize/optimize.md). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). + Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../sql-reference/statements/optimize.md). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). [Original article](https://clickhouse.tech/docs/en/operations/table_engines/graphitemergetree/) From ea7dc495cf87f971e10e64961a667bd3e35f2e86 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 17 Aug 2021 12:32:49 +0800 Subject: [PATCH 107/220] Better code. --- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 32 +++++++----------- .../MergeTree/MergeTreeDataSelectExecutor.h | 2 +- .../MergeTree/StorageFromMergeTreeDataPart.h | 33 +++++++++---------- src/Storages/ReadInOrderOptimizer.cpp | 2 -- 4 files changed, 28 insertions(+), 41 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ff0c0657fd9..f5c1890154a 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -133,11 +133,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( std::shared_ptr max_block_numbers_to_read) const { const auto & settings = context->getSettingsRef(); - auto parts = data.getDataPartsVector(); if (!query_info.projection) { auto plan = readFromParts( - parts, + query_info.merge_tree_select_result_ptr ? MergeTreeData::DataPartsVector{} : data.getDataPartsVector(), column_names_to_return, metadata_snapshot, metadata_snapshot, @@ -163,27 +162,15 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( ProjectionDescription::typeToString(query_info.projection->desc->type), query_info.projection->desc->name); - MergeTreeData::DataPartsVector projection_parts; - MergeTreeData::DataPartsVector normal_parts; - for (const auto & part : parts) - { - const auto & projections = part->getProjectionParts(); - auto it = projections.find(query_info.projection->desc->name); - if (it != projections.end()) - projection_parts.push_back(it->second); - else - normal_parts.push_back(part); - } - Pipes pipes; Pipe projection_pipe; Pipe ordinary_pipe; - if (!projection_parts.empty()) + if (query_info.projection->merge_tree_projection_select_result_ptr) { LOG_DEBUG(log, "projection required columns: {}", fmt::join(query_info.projection->required_columns, ", ")); auto plan = readFromParts( - projection_parts, + {}, query_info.projection->required_columns, metadata_snapshot, query_info.projection->desc->metadata, @@ -224,10 +211,10 @@ QueryPlanPtr MergeTreeDataSelectExecutor::read( } } - if (!normal_parts.empty()) + if (query_info.projection->merge_tree_normal_select_result_ptr) { auto storage_from_base_parts_of_projection - = StorageFromMergeTreeDataPart::create(std::move(normal_parts), query_info.projection->merge_tree_normal_select_result_ptr); + = StorageFromMergeTreeDataPart::create(data, query_info.projection->merge_tree_normal_select_result_ptr); auto interpreter = InterpreterSelectQuery( query_info.query, context, @@ -1133,8 +1120,13 @@ QueryPlanPtr MergeTreeDataSelectExecutor::readFromParts( std::shared_ptr max_block_numbers_to_read, MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr) const { - size_t total_parts = parts.size(); - if (total_parts == 0) + /// If merge_tree_select_result_ptr != nullptr, we use analyzed result so parts will always be empty. + if (merge_tree_select_result_ptr) + { + if (merge_tree_select_result_ptr->marks() == 0) + return std::make_unique(); + } + else if (parts.empty()) return std::make_unique(); Names real_column_names; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index f8f50723ff0..92c4382dc90 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -47,7 +47,7 @@ public: UInt64 max_block_size, unsigned num_streams, std::shared_ptr max_block_numbers_to_read = nullptr, - MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr = nullptr) const; + MergeTreeDataSelectAnalysisResultPtr merge_tree_select_result_ptr = nullptr) const; /// Get an estimation for the number of marks we are going to read. /// Reads nothing. Secondary indexes are not used. diff --git a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h index 26df2e6d658..bcce2d990ca 100644 --- a/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h +++ b/src/Storages/MergeTree/StorageFromMergeTreeDataPart.h @@ -31,8 +31,7 @@ public: size_t max_block_size, unsigned num_streams) override { - // NOTE: It's used to read normal parts only - QueryPlan query_plan = std::move(*MergeTreeDataSelectExecutor(parts.front()->storage) + QueryPlan query_plan = std::move(*MergeTreeDataSelectExecutor(storage) .readFromParts( parts, column_names, @@ -56,41 +55,46 @@ public: bool mayBenefitFromIndexForIn( const ASTPtr & left_in_operand, ContextPtr query_context, const StorageMetadataPtr & metadata_snapshot) const override { - return parts.front()->storage.mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot); + return storage.mayBenefitFromIndexForIn(left_in_operand, query_context, metadata_snapshot); } NamesAndTypesList getVirtuals() const override { - return parts.front()->storage.getVirtuals(); + return storage.getVirtuals(); } String getPartitionId() const { - return parts.front()->info.partition_id; + return partition_id; } String getPartitionIDFromQuery(const ASTPtr & ast, ContextPtr context) const { - return parts.front()->storage.getPartitionIDFromQuery(ast, context); + return storage.getPartitionIDFromQuery(ast, context); } protected: + /// Used in part mutation. StorageFromMergeTreeDataPart(const MergeTreeData::DataPartPtr & part_) : IStorage(getIDFromPart(part_)) , parts({part_}) + , storage(part_->storage) + , partition_id(part_->info.partition_id) { - setInMemoryMetadata(part_->storage.getInMemoryMetadata()); + setInMemoryMetadata(storage.getInMemoryMetadata()); } - StorageFromMergeTreeDataPart( - MergeTreeData::DataPartsVector && parts_, MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr_ = nullptr) - : IStorage(getIDFromParts(parts_)), parts(std::move(parts_)), analysis_result_ptr(analysis_result_ptr_) + /// Used in queries with projection. + StorageFromMergeTreeDataPart(const MergeTreeData & storage_, MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr_) + : IStorage(storage_.getStorageID()), storage(storage_), analysis_result_ptr(analysis_result_ptr_) { - setInMemoryMetadata(parts.front()->storage.getInMemoryMetadata()); + setInMemoryMetadata(storage.getInMemoryMetadata()); } private: MergeTreeData::DataPartsVector parts; + const MergeTreeData & storage; + String partition_id; MergeTreeDataSelectAnalysisResultPtr analysis_result_ptr; static StorageID getIDFromPart(const MergeTreeData::DataPartPtr & part_) @@ -98,13 +102,6 @@ private: auto table_id = part_->storage.getStorageID(); return StorageID(table_id.database_name, table_id.table_name + " (part " + part_->name + ")"); } - - static StorageID getIDFromParts(const MergeTreeData::DataPartsVector & parts_) - { - assert(!parts_.empty()); - auto table_id = parts_.front()->storage.getStorageID(); - return StorageID(table_id.database_name, table_id.table_name + " (parts)"); - } }; } diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index 912d284bfc0..bae24f97b28 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -6,8 +6,6 @@ #include #include #include -#include -#include namespace DB { From 6001c7f214d48ce028eeceeecd34dc5248c410a6 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 17 Aug 2021 08:44:51 +0300 Subject: [PATCH 108/220] Update InterpreterSelectQuery.cpp --- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 9f4d573735e..bbcd2db27d9 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -890,7 +890,7 @@ static bool hasWithTotalsInAnySubqueryInFromClause(const ASTSelectQuery & query) } /// After normalization, the height of the AST tree can have any depth, /// but the number of children at each level is always 2. - else if (const auto * child_intersect_except = elem->as()) + else if (elem->as()) { std::function traverse_recursively = [&](ASTPtr child_ast) -> bool { From 6ea1bd1e8cabda9a602e0bd4a8d131a0220300bf Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 17 Aug 2021 09:02:37 +0300 Subject: [PATCH 109/220] More tests --- .../02009_decimal_no_trailing_zeros.reference | 32 +++++++++---------- .../02009_decimal_no_trailing_zeros.sql | 19 +++++++++++ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference index d41682b62ce..7986d51bfda 100644 --- a/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference +++ b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference @@ -1,32 +1,30 @@ --- { echo } - -SELECT 1.123::Decimal64(1); 1.1 -SELECT 1.123::Decimal64(2); 1.12 -SELECT 1.123::Decimal64(3); 1.123 -SELECT 1.123::Decimal64(4); 1.123 -SELECT 1.123::Decimal64(5); 1.123 -SELECT 1.123::Decimal64(10); 1.123 -SELECT 1::Decimal64(0); 1 -SELECT 1::Decimal64(1); 1 -SELECT 1::Decimal64(10); 1 -SELECT 1.1234567::Decimal32(8); 1.1234567 -SELECT 1.1234567890::Decimal64(10); 1.123456789 -SELECT 1.1234567890::Decimal128(10); 1.123456789 -SELECT 1.1234567890::Decimal256(10); 1.123456789 -SELECT 1.123456789012345678901::Decimal256(20); 1.1234567890123456789 -SELECT 1.123456789012345678901::Decimal256(22); 1.123456789012345678901 +1.1 +1.12 +1.123 +1.1230 +1.12300 +1.1230000000 +1 +1.0 +1.0000000000 +1.12345670 +1.1234567890 +1.1234567890 +1.1234567890 +1.12345678901234567890 +1.1234567890123456789010 diff --git a/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.sql b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.sql index 556e355e7d8..e88e878b378 100644 --- a/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.sql +++ b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.sql @@ -16,3 +16,22 @@ SELECT 1.1234567890::Decimal128(10); SELECT 1.1234567890::Decimal256(10); SELECT 1.123456789012345678901::Decimal256(20); SELECT 1.123456789012345678901::Decimal256(22); + +SET output_format_decimal_trailing_zeros = 1; + +SELECT 1.123::Decimal64(1); +SELECT 1.123::Decimal64(2); +SELECT 1.123::Decimal64(3); +SELECT 1.123::Decimal64(4); +SELECT 1.123::Decimal64(5); +SELECT 1.123::Decimal64(10); +SELECT 1::Decimal64(0); +SELECT 1::Decimal64(1); +SELECT 1::Decimal64(10); + +SELECT 1.1234567::Decimal32(8); +SELECT 1.1234567890::Decimal64(10); +SELECT 1.1234567890::Decimal128(10); +SELECT 1.1234567890::Decimal256(10); +SELECT 1.123456789012345678901::Decimal256(20); +SELECT 1.123456789012345678901::Decimal256(22); From 4051db8eabfe5aec12c4f2ab8d7b890e0f2cfcc0 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 17 Aug 2021 09:11:35 +0300 Subject: [PATCH 110/220] Add performance test --- tests/performance/decimal_format.xml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tests/performance/decimal_format.xml diff --git a/tests/performance/decimal_format.xml b/tests/performance/decimal_format.xml new file mode 100644 index 00000000000..6841ea93d12 --- /dev/null +++ b/tests/performance/decimal_format.xml @@ -0,0 +1,3 @@ + + SELECT count() FROM zeros(10000000) WHERE NOT ignore(toString((rand() / 1000000)::Decimal64(6))) + From af92616cdcdf0678f0dbb7275fb433e94226b427 Mon Sep 17 00:00:00 2001 From: Nikolay Degterinsky Date: Tue, 17 Aug 2021 10:00:24 +0300 Subject: [PATCH 111/220] Add test --- tests/queries/0_stateless/02011_http_parsing.reference | 6 ++++++ tests/queries/0_stateless/02011_http_parsing.sh | 10 ++++++++++ 2 files changed, 16 insertions(+) create mode 100644 tests/queries/0_stateless/02011_http_parsing.reference create mode 100755 tests/queries/0_stateless/02011_http_parsing.sh diff --git a/tests/queries/0_stateless/02011_http_parsing.reference b/tests/queries/0_stateless/02011_http_parsing.reference new file mode 100644 index 00000000000..e059b92b8b7 --- /dev/null +++ b/tests/queries/0_stateless/02011_http_parsing.reference @@ -0,0 +1,6 @@ +One +Two +Three +Four +Five +Six diff --git a/tests/queries/0_stateless/02011_http_parsing.sh b/tests/queries/0_stateless/02011_http_parsing.sh new file mode 100755 index 00000000000..8021f7c550b --- /dev/null +++ b/tests/queries/0_stateless/02011_http_parsing.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo -ne 'One\nTwo\n' | ${CLICKHOUSE_CURL} -sSF 'metrics_list=@-;' "${CLICKHOUSE_URL}/?metrics_list_format=TSV&metrics_list_structure=Path+String&query=SELECT+*+FROM+metrics_list"; +echo -ne 'Three\nFour' | ${CLICKHOUSE_CURL} -sSF 'metrics_list=@-;' "${CLICKHOUSE_URL}/?metrics_list_format=TSV&metrics_list_structure=Path+String&query=SELECT+*+FROM+metrics_list"; +echo -ne 'Five\n' | ${CLICKHOUSE_CURL} -sSF 'metrics_list=@-;' "${CLICKHOUSE_URL}/?metrics_list_format=TSV&metrics_list_structure=Path+String&query=SELECT+*+FROM+metrics_list"; +echo -ne 'Six' | ${CLICKHOUSE_CURL} -sSF 'metrics_list=@-;' "${CLICKHOUSE_URL}/?metrics_list_format=TSV&metrics_list_structure=Path+String&query=SELECT+*+FROM+metrics_list"; From 1a3b72fa09b63be64e6e3ff734279fd01997ba33 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Tue, 17 Aug 2021 10:15:17 +0300 Subject: [PATCH 112/220] Update test --- .../02009_decimal_no_trailing_zeros.reference | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference index 7986d51bfda..58f5180322a 100644 --- a/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference +++ b/tests/queries/0_stateless/02009_decimal_no_trailing_zeros.reference @@ -1,30 +1,63 @@ +-- { echo } + +SELECT 1.123::Decimal64(1); 1.1 +SELECT 1.123::Decimal64(2); 1.12 +SELECT 1.123::Decimal64(3); 1.123 +SELECT 1.123::Decimal64(4); 1.123 +SELECT 1.123::Decimal64(5); 1.123 +SELECT 1.123::Decimal64(10); 1.123 +SELECT 1::Decimal64(0); 1 +SELECT 1::Decimal64(1); 1 +SELECT 1::Decimal64(10); 1 +SELECT 1.1234567::Decimal32(8); 1.1234567 +SELECT 1.1234567890::Decimal64(10); 1.123456789 +SELECT 1.1234567890::Decimal128(10); 1.123456789 +SELECT 1.1234567890::Decimal256(10); 1.123456789 +SELECT 1.123456789012345678901::Decimal256(20); 1.1234567890123456789 +SELECT 1.123456789012345678901::Decimal256(22); 1.123456789012345678901 +SET output_format_decimal_trailing_zeros = 1; +SELECT 1.123::Decimal64(1); 1.1 +SELECT 1.123::Decimal64(2); 1.12 +SELECT 1.123::Decimal64(3); 1.123 +SELECT 1.123::Decimal64(4); 1.1230 +SELECT 1.123::Decimal64(5); 1.12300 +SELECT 1.123::Decimal64(10); 1.1230000000 +SELECT 1::Decimal64(0); 1 +SELECT 1::Decimal64(1); 1.0 +SELECT 1::Decimal64(10); 1.0000000000 +SELECT 1.1234567::Decimal32(8); 1.12345670 +SELECT 1.1234567890::Decimal64(10); 1.1234567890 +SELECT 1.1234567890::Decimal128(10); 1.1234567890 +SELECT 1.1234567890::Decimal256(10); 1.1234567890 +SELECT 1.123456789012345678901::Decimal256(20); 1.12345678901234567890 +SELECT 1.123456789012345678901::Decimal256(22); 1.1234567890123456789010 From dc884cfe05875492f2a3b2f88775b557d365f244 Mon Sep 17 00:00:00 2001 From: igomac <714541080@qq.com> Date: Tue, 17 Aug 2021 15:45:53 +0800 Subject: [PATCH 113/220] Update backup.md Fix anchor link --- docs/zh/operations/backup.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/zh/operations/backup.md b/docs/zh/operations/backup.md index 6d517e6ccb3..4809820798c 100644 --- a/docs/zh/operations/backup.md +++ b/docs/zh/operations/backup.md @@ -7,7 +7,7 @@ toc_title: "\u6570\u636E\u5907\u4EFD" # 数据备份 {#data-backup} -尽管 [副本] (../engines/table-engines/mergetree-family/replication.md) 可以提供针对硬件的错误防护, 但是它不能预防人为操作失误: 数据的意外删除, 错误表的删除或者错误集群上表的删除, 以及导致错误数据处理或者数据损坏的软件bug. 在很多案例中，这类意外可能会影响所有的副本. ClickHouse 有内置的保护措施可以预防一些错误 — 例如, 默认情况下 [不能人工删除使用带有MergeTree引擎且包含超过50Gb数据的表] (server-configuration-parameters/settings.md#max-table-size-to-drop). 但是，这些保护措施不能覆盖所有可能情况，并且这些措施可以被绕过。 +尽管 [副本](../engines/table-engines/mergetree-family/replication.md) 可以提供针对硬件的错误防护, 但是它不能预防人为操作失误: 数据的意外删除, 错误表的删除或者错误集群上表的删除, 以及导致错误数据处理或者数据损坏的软件bug. 在很多案例中，这类意外可能会影响所有的副本. ClickHouse 有内置的保护措施可以预防一些错误 — 例如, 默认情况下 [不能人工删除使用带有MergeTree引擎且包含超过50Gb数据的表](server-configuration-parameters/settings.md#max-table-size-to-drop). 但是，这些保护措施不能覆盖所有可能情况，并且这些措施可以被绕过。 为了有效地减少可能的人为错误，您应该 **提前** 仔细的准备备份和数据还原的策略. @@ -18,26 +18,26 @@ toc_title: "\u6570\u636E\u5907\u4EFD" ## 将源数据复制到其它地方 {#duplicating-source-data-somewhere-else} -通常摄入到ClickHouse的数据是通过某种持久队列传递的，例如 [Apache Kafka] (https://kafka.apache.org). 在这种情况下，可以配置一组额外的订阅服务器，这些订阅服务器将在写入ClickHouse时读取相同的数据流，并将其存储在冷存储中。 大多数公司已经有一些默认推荐的冷存储，可能是对象存储或分布式文件系统，如 [HDFS] (https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). +通常摄入到ClickHouse的数据是通过某种持久队列传递的，例如 [Apache Kafka](https://kafka.apache.org). 在这种情况下，可以配置一组额外的订阅服务器，这些订阅服务器将在写入ClickHouse时读取相同的数据流，并将其存储在冷存储中。 大多数公司已经有一些默认推荐的冷存储，可能是对象存储或分布式文件系统，如 [HDFS](https://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html). ## 文件系统快照 {#filesystem-snapshots} -某些本地文件系统提供快照功能（例如, [ZFS] (https://en.wikipedia.org/wiki/ZFS)），但它们可能不是提供实时查询的最佳选择。 一个可能的解决方案是使用这种文件系统创建额外的副本，并将它们与用于`SELECT` 查询的 [分布式] (../engines/table-engines/special/distributed.md) 表分离。 任何修改数据的查询都无法访问此类副本上的快照。 作为回报，这些副本可能具有特殊的硬件配置，每个服务器附加更多的磁盘，这将是经济高效的。 +某些本地文件系统提供快照功能（例如, [ZFS](https://en.wikipedia.org/wiki/ZFS)），但它们可能不是提供实时查询的最佳选择。 一个可能的解决方案是使用这种文件系统创建额外的副本，并将它们与用于`SELECT` 查询的 [分布式](../engines/table-engines/special/distributed.md) 表分离。 任何修改数据的查询都无法访问此类副本上的快照。 作为回报，这些副本可能具有特殊的硬件配置，每个服务器附加更多的磁盘，这将是经济高效的。 ## clickhouse-copier {#clickhouse-copier} -[clickhouse-copier] (utilities/clickhouse-copier.md) 是一个多功能工具，最初创建它是为了用于重新切分pb大小的表。 因为它能够在ClickHouse表和集群之间可靠地复制数据，所以它也可用于备份和还原数据。 +[clickhouse-copier](utilities/clickhouse-copier.md) 是一个多功能工具，最初创建它是为了用于重新切分pb大小的表。 因为它能够在ClickHouse表和集群之间可靠地复制数据，所以它也可用于备份和还原数据。 对于较小的数据量，一个简单的 `INSERT INTO ... SELECT ...` 到远程表也可以工作。 ## part操作 {#manipulations-with-parts} -ClickHouse允许使用 `ALTER TABLE ... FREEZE PARTITION ...` 查询以创建表分区的本地副本。 这是利用硬链接(hardlink)到 `/var/lib/clickhouse/shadow/` 文件夹中实现的，所以它通常不会因为旧数据而占用额外的磁盘空间。 创建的文件副本不由ClickHouse服务器处理，所以你可以把它们留在那里：你将有一个简单的备份，不需要任何额外的外部系统，但它仍然容易出现硬件问题。 出于这个原因，最好将它们远程复制到另一个位置，然后删除本地副本。 分布式文件系统和对象存储仍然是一个不错的选择，但是具有足够大容量的正常附加文件服务器也可以工作（在这种情况下，传输将通过网络文件系统或者也许是 [rsync] (https://en.wikipedia.org/wiki/Rsync) 来进行). +ClickHouse允许使用 `ALTER TABLE ... FREEZE PARTITION ...` 查询以创建表分区的本地副本。 这是利用硬链接(hardlink)到 `/var/lib/clickhouse/shadow/` 文件夹中实现的，所以它通常不会因为旧数据而占用额外的磁盘空间。 创建的文件副本不由ClickHouse服务器处理，所以你可以把它们留在那里：你将有一个简单的备份，不需要任何额外的外部系统，但它仍然容易出现硬件问题。 出于这个原因，最好将它们远程复制到另一个位置，然后删除本地副本。 分布式文件系统和对象存储仍然是一个不错的选择，但是具有足够大容量的正常附加文件服务器也可以工作（在这种情况下，传输将通过网络文件系统或者也许是 [rsync](https://en.wikipedia.org/wiki/Rsync) 来进行). 数据可以使用 `ALTER TABLE ... ATTACH PARTITION ...` 从备份中恢复。 -有关与分区操作相关的查询的详细信息，请参阅 [更改文档] (../sql-reference/statements/alter.md#alter_manipulations-with-partitions). +有关与分区操作相关的查询的详细信息，请参阅 [更改文档](../sql-reference/statements/alter.md#alter_manipulations-with-partitions). -第三方工具可用于自动化此方法: [clickhouse-backup] (https://github.com/AlexAkulov/clickhouse-backup). +第三方工具可用于自动化此方法: [clickhouse-backup](https://github.com/AlexAkulov/clickhouse-backup). -[原始文章] (https://clickhouse.tech/docs/en/operations/backup/) +[原始文章](https://clickhouse.tech/docs/en/operations/backup/) From c280d3ba029421f143ff2b742bae2ccf344423a5 Mon Sep 17 00:00:00 2001 From: igomac <714541080@qq.com> Date: Tue, 17 Aug 2021 15:47:33 +0800 Subject: [PATCH 114/220] Update data_type_families.md Special words are not translated --- docs/zh/operations/system-tables/data_type_families.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/operations/system-tables/data_type_families.md b/docs/zh/operations/system-tables/data_type_families.md index 000abfce65d..d4ee7efa034 100644 --- a/docs/zh/operations/system-tables/data_type_families.md +++ b/docs/zh/operations/system-tables/data_type_families.md @@ -3,7 +3,7 @@ machine_translated: true machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 --- -# 系统。data_type_families {#system_tables-data_type_families} +# system.data_type_families {#system_tables-data_type_families} 包含有关受支持的[数据类型](../../sql-reference/data-types/)的信息. From 2239ae91a374d4b51ba1fbffc9b26f06508723ea Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 17 Aug 2021 10:57:13 +0300 Subject: [PATCH 115/220] Fix --- src/Parsers/ASTSelectWithUnionQuery.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index fa7359574f8..f0ac9cbf3ed 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -32,12 +32,15 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F auto mode_to_str = [&](auto mode) { - if (mode == Mode::Unspecified) - return ""; - else if (mode == Mode::ALL) + if (mode == Mode::ALL) return " ALL"; - else + else if (mode == Mode::DISTINCT) return " DISTINCT"; + else if (mode == Mode::INTERSECT) + return " INTERSECT"; + else if (mode == Mode::EXCEPT) + return " EXCEPT"; + return ""; }; for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) From 8da7f406afb541206f6a3fc11514e1a214aa7891 Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 17 Aug 2021 11:03:01 +0300 Subject: [PATCH 116/220] Better comment --- src/Interpreters/InterpreterSelectQuery.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index bbcd2db27d9..297e41c9546 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -881,15 +881,16 @@ static bool hasWithTotalsInAnySubqueryInFromClause(const ASTSelectQuery & query) { for (const auto & elem : ast_union->list_of_selects->children) { - /// After normalization, the height of the AST tree is at most 2. + /// After normalization for union child node the height of the AST tree is at most 2. if (const auto * child_union = elem->as()) { for (const auto & child_elem : child_union->list_of_selects->children) if (hasWithTotalsInAnySubqueryInFromClause(child_elem->as())) return true; } - /// After normalization, the height of the AST tree can have any depth, - /// but the number of children at each level is always 2. + /// After normalization in case there are intersect or except nodes, the height of + /// the AST tree can have any depth (each intersect/except adds a level), but the + /// number of children in those nodes is always 2. else if (elem->as()) { std::function traverse_recursively = [&](ASTPtr child_ast) -> bool From c7dc42e30b6aca0199e296ebf86ddf7a5ef2b2dc Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Tue, 17 Aug 2021 11:11:28 +0300 Subject: [PATCH 117/220] Update 00597_push_down_predicate_long.sql --- tests/queries/0_stateless/00597_push_down_predicate_long.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/00597_push_down_predicate_long.sql b/tests/queries/0_stateless/00597_push_down_predicate_long.sql index 8096cbef46b..2e3357241ad 100644 --- a/tests/queries/0_stateless/00597_push_down_predicate_long.sql +++ b/tests/queries/0_stateless/00597_push_down_predicate_long.sql @@ -8,8 +8,7 @@ DROP TABLE IF EXISTS test_view_00597; CREATE TABLE test_00597(date Date, id Int8, name String, value Int64) ENGINE = MergeTree(date, (id, date), 8192); CREATE VIEW test_view_00597 AS SELECT * FROM test_00597; --- TODO: This query should execute successfully: -SELECT * FROM (SELECT floor(floor(1, floor(NULL), id = 257), floor(floor(floor(floor(NULL), '10485.76', '9223372036854775807', NULL), floor(10, floor(65535, NULL), 100.0000991821289), NULL)), '2.56'), b.* FROM (SELECT floor(floor(floor(floor(NULL), 1000.0001220703125))), * FROM test_00597) AS b) WHERE id = 257; -- { serverError 352 } +SELECT * FROM (SELECT floor(floor(1, floor(NULL), id = 257), floor(floor(floor(floor(NULL), '10485.76', '9223372036854775807', NULL), floor(10, floor(65535, NULL), 100.0000991821289), NULL)), '2.56'), b.* FROM (SELECT floor(floor(floor(floor(NULL), 1000.0001220703125))), * FROM test_00597) AS b) WHERE id = 257; INSERT INTO test_00597 VALUES('2000-01-01', 1, 'test string 1', 1); INSERT INTO test_00597 VALUES('2000-01-01', 2, 'test string 2', 2); From 77e8db92ba4852cfd31d0d0abda97387a3f19eee Mon Sep 17 00:00:00 2001 From: igomac <714541080@qq.com> Date: Tue, 17 Aug 2021 16:47:34 +0800 Subject: [PATCH 118/220] Update backup.md remove machine_translated mark --- docs/zh/operations/backup.md | 6 ------ 1 file changed, 6 deletions(-) diff --git a/docs/zh/operations/backup.md b/docs/zh/operations/backup.md index 4809820798c..75fbd83089a 100644 --- a/docs/zh/operations/backup.md +++ b/docs/zh/operations/backup.md @@ -1,9 +1,3 @@ ---- -machine_translated: true -machine_translated_rev: 72537a2d527c63c07aa5d2361a8829f3895cf2bd -toc_priority: 49 -toc_title: "\u6570\u636E\u5907\u4EFD" ---- # 数据备份 {#data-backup} From 8660b17bf4fe52227ef33a14f5e79775d1af0d0e Mon Sep 17 00:00:00 2001 From: igomac <714541080@qq.com> Date: Tue, 17 Aug 2021 16:48:55 +0800 Subject: [PATCH 119/220] Update data_type_families.md remove machine_translated mark --- docs/zh/operations/system-tables/data_type_families.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/zh/operations/system-tables/data_type_families.md b/docs/zh/operations/system-tables/data_type_families.md index d4ee7efa034..b1d114a6df0 100644 --- a/docs/zh/operations/system-tables/data_type_families.md +++ b/docs/zh/operations/system-tables/data_type_families.md @@ -1,7 +1,3 @@ ---- -machine_translated: true -machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 ---- # system.data_type_families {#system_tables-data_type_families} From 0950584f8824a2f8f77d4bc3053855ca6bfad8f8 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Tue, 17 Aug 2021 11:49:02 +0300 Subject: [PATCH 120/220] Update ASTSelectWithUnionQuery.cpp --- src/Parsers/ASTSelectWithUnionQuery.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Parsers/ASTSelectWithUnionQuery.cpp b/src/Parsers/ASTSelectWithUnionQuery.cpp index f0ac9cbf3ed..e31dea83d82 100644 --- a/src/Parsers/ASTSelectWithUnionQuery.cpp +++ b/src/Parsers/ASTSelectWithUnionQuery.cpp @@ -33,20 +33,20 @@ void ASTSelectWithUnionQuery::formatQueryImpl(const FormatSettings & settings, F auto mode_to_str = [&](auto mode) { if (mode == Mode::ALL) - return " ALL"; + return "UNION ALL"; else if (mode == Mode::DISTINCT) - return " DISTINCT"; + return "UNION DISTINCT"; else if (mode == Mode::INTERSECT) - return " INTERSECT"; + return "INTERSECT"; else if (mode == Mode::EXCEPT) - return " EXCEPT"; + return "EXCEPT"; return ""; }; for (ASTs::const_iterator it = list_of_selects->children.begin(); it != list_of_selects->children.end(); ++it) { if (it != list_of_selects->children.begin()) - settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << "UNION" + settings.ostr << settings.nl_or_ws << indent_str << (settings.hilite ? hilite_keyword : "") << mode_to_str((is_normalized) ? union_mode : list_of_modes[it - list_of_selects->children.begin() - 1]) << (settings.hilite ? hilite_none : ""); From 84660f36de124ad5a6480a945ae8968f8381c3a3 Mon Sep 17 00:00:00 2001 From: Sergei Semin Date: Tue, 17 Aug 2021 12:17:47 +0300 Subject: [PATCH 121/220] add -Wno-reserved-identifier in necessary places --- utils/corrector_utf8/CMakeLists.txt | 1 + utils/iotest/CMakeLists.txt | 2 ++ utils/zookeeper-cli/CMakeLists.txt | 1 + 3 files changed, 4 insertions(+) diff --git a/utils/corrector_utf8/CMakeLists.txt b/utils/corrector_utf8/CMakeLists.txt index 4784fd43e2d..a426815bf99 100644 --- a/utils/corrector_utf8/CMakeLists.txt +++ b/utils/corrector_utf8/CMakeLists.txt @@ -1,2 +1,3 @@ add_executable(corrector_utf8 corrector_utf8.cpp) target_link_libraries(corrector_utf8 PRIVATE clickhouse_common_io) +target_no_warning(corrector_utf8 reserved-identifier) diff --git a/utils/iotest/CMakeLists.txt b/utils/iotest/CMakeLists.txt index 8f141b178f0..66e2b982104 100644 --- a/utils/iotest/CMakeLists.txt +++ b/utils/iotest/CMakeLists.txt @@ -4,6 +4,8 @@ target_link_libraries (iotest PRIVATE clickhouse_common_io) add_executable (iotest_nonblock iotest_nonblock.cpp ${SRCS}) target_link_libraries (iotest_nonblock PRIVATE clickhouse_common_io) +target_no_warning(iotest_nonblock reserved-identifier) add_executable (iotest_aio iotest_aio.cpp ${SRCS}) target_link_libraries (iotest_aio PRIVATE clickhouse_common_io) +target_no_warning(iotest_aio reserved-identifier) diff --git a/utils/zookeeper-cli/CMakeLists.txt b/utils/zookeeper-cli/CMakeLists.txt index 2199a1b38ff..90794dcceb5 100644 --- a/utils/zookeeper-cli/CMakeLists.txt +++ b/utils/zookeeper-cli/CMakeLists.txt @@ -1,2 +1,3 @@ add_executable(clickhouse-zookeeper-cli zookeeper-cli.cpp) target_link_libraries(clickhouse-zookeeper-cli PRIVATE clickhouse_common_zookeeper) +target_no_warning(clickhouse-zookeeper-cli reserved-identifier) From 27ff08140324b79d237bc1c70c28726e6a312578 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Tue, 17 Aug 2021 16:46:47 +0800 Subject: [PATCH 122/220] Fix some leftover TODOs --- src/Client/MultiplexedConnections.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index fe3879fdd30..a4e1eb09253 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -373,7 +373,9 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead except_list, is_draining ? drain_timeout : receive_timeout); - if (n == 0) + /// We treat any error as timeout for simplicity. + /// And we also check if read_list is still empty just in case. + if (n <= 0 || read_list.empty()) { auto err_msg = fmt::format("Timeout exceeded while reading from {}", dumpAddressesUnlocked()); for (ReplicaState & state : replica_states) @@ -389,9 +391,7 @@ MultiplexedConnections::ReplicaState & MultiplexedConnections::getReplicaForRead } } - /// TODO Absolutely wrong code: read_list could be empty; motivation of rand is unclear. - /// This code path is disabled by default. - + /// TODO Motivation of rand is unclear. auto & socket = read_list[thread_local_rng() % read_list.size()]; if (fd_to_replica_state_idx.empty()) { From c9404c1fb89f13ef984173e371e45bf313cf4340 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Tue, 17 Aug 2021 13:30:51 +0300 Subject: [PATCH 123/220] Update entrypoint.sh --- docker/test/performance-comparison/entrypoint.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index fd25a673c85..1d03d953ccd 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -146,9 +146,11 @@ dmesg -T > dmesg.log cat /proc/sys/kernel/core_pattern +ls -lath + 7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} \ {right,left}/{performance,scripts} {{right,left}/db,db0}/preprocessed_configs \ report analyze benchmark metrics \ - ./*.core + ./*.core.dmp cp compare.log /output From afa748c717a4d38503dddeee10d5e7c95916f36b Mon Sep 17 00:00:00 2001 From: vdimir Date: Fri, 6 Aug 2021 17:15:11 +0300 Subject: [PATCH 124/220] Refactor NotJoined pt1 --- src/Interpreters/HashJoin.cpp | 67 +++++++------------ src/Interpreters/MergeJoin.cpp | 67 +++++-------------- src/Interpreters/MergeJoin.h | 1 + src/Interpreters/TableJoin.cpp | 18 +++++ src/Interpreters/TableJoin.h | 1 + src/Interpreters/join_common.cpp | 58 ++++++---------- src/Interpreters/join_common.h | 60 ++++++++++++----- .../Transforms/JoiningTransform.cpp | 1 - 8 files changed, 125 insertions(+), 148 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index e0aad706966..4130431485e 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -1468,40 +1468,23 @@ struct AdderNonJoined /// Stream from not joined earlier rows of the right table. -class NonJoinedBlockInputStream : private NotJoined, public IBlockInputStream +class NonJoinedBlockInputStream final : public NotJoined { public: - NonJoinedBlockInputStream(const HashJoin & parent_, const Block & result_sample_block_, UInt64 max_block_size_) - : NotJoined(*parent_.table_join, - parent_.savedBlockSample(), - parent_.right_sample_block, - result_sample_block_) + NonJoinedBlockInputStream( + const HashJoin & parent_, + const Block & result_sample_block_, + size_t left_columns_count, + UInt64 max_block_size_) + : NotJoined(parent_.savedBlockSample(), result_sample_block_, + left_columns_count, parent_.table_join->leftToRightKeyRemap()) , parent(parent_) , max_block_size(max_block_size_) {} - String getName() const override { return "NonJoined"; } - Block getHeader() const override { return result_sample_block; } - protected: - Block readImpl() override + size_t fillColumns(MutableColumns & columns_right) override { - if (parent.data->blocks.empty()) - return Block(); - return createBlock(); - } - -private: - const HashJoin & parent; - UInt64 max_block_size; - - std::any position; - std::optional nulls_position; - - Block createBlock() - { - MutableColumns columns_right = saved_block_sample.cloneEmptyColumns(); - size_t rows_added = 0; auto fill_callback = [&](auto, auto strictness, auto & map) @@ -1513,22 +1496,16 @@ private: throw Exception("Logical error: unknown JOIN strictness (must be on of: ANY, ALL, ASOF)", ErrorCodes::LOGICAL_ERROR); fillNullsFromBlocks(columns_right, rows_added); - if (!rows_added) - return {}; - - Block res = result_sample_block.cloneEmpty(); - addLeftColumns(res, rows_added); - addRightColumns(res, columns_right); - copySameKeys(res); - correctLowcardAndNullability(res); - -#ifndef NDEBUG - assertBlocksHaveEqualStructure(res, result_sample_block, getName()); -#endif - - return res; + return rows_added; } +private: + const HashJoin & parent; + UInt64 max_block_size; + + std::any position; + std::optional nulls_position; + template size_t fillColumnsFromMap(const Maps & maps, MutableColumns & columns_keys_and_right) { @@ -1610,12 +1587,14 @@ private: BlockInputStreamPtr HashJoin::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const { if (table_join->strictness() == ASTTableJoin::Strictness::Asof || - table_join->strictness() == ASTTableJoin::Strictness::Semi) + table_join->strictness() == ASTTableJoin::Strictness::Semi || + !isRightOrFull(table_join->kind())) + { return {}; + } - if (isRightOrFull(table_join->kind())) - return std::make_shared(*this, result_sample_block, max_block_size); - return {}; + size_t left_columns_count = result_sample_block.columns() - required_right_keys.columns() - sample_block_with_columns_to_add.columns(); + return std::make_shared(*this, result_sample_block, left_columns_count, max_block_size); } void HashJoin::reuseJoinedData(const HashJoin & join) diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index a2c63a4693b..84d5a80cff5 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -1,7 +1,8 @@ #include #include -#include +#include + #include #include #include @@ -723,15 +724,7 @@ void MergeJoin::joinBlock(Block & block, ExtraBlockPtr & not_processed) if (needConditionJoinColumn()) block.erase(deriveTempName(mask_column_name_left)); - for (const auto & column_name : lowcard_keys) - { - if (!block.has(column_name)) - continue; - if (auto & col = block.getByName(column_name); !col.type->lowCardinality()) - JoinCommon::changeLowCardinalityInplace(col); - } - - JoinCommon::restoreLowCardinalityInplace(block); + JoinCommon::restoreLowCardinalityInplace(block, lowcard_keys); } template @@ -1035,55 +1028,25 @@ void MergeJoin::initRightTableWriter() } /// Stream from not joined earlier rows of the right table. -class NonMergeJoinedBlockInputStream : private NotJoined, public IBlockInputStream +class NonMergeJoinedBlockInputStream final : public NotJoined { public: NonMergeJoinedBlockInputStream(const MergeJoin & parent_, - const Block & result_sample_block_, - const Names & key_names_right_, + const Block & result_sample_block, + size_t left_columns_count, UInt64 max_block_size_) - : NotJoined(*parent_.table_join, - parent_.modifyRightBlock(parent_.right_sample_block), - parent_.right_sample_block, - result_sample_block_, - {}, key_names_right_) + : NotJoined(parent_.modifyRightBlock(parent_.right_sample_block), + result_sample_block, + left_columns_count, + parent_.table_join->leftToRightKeyRemap()) , parent(parent_) , max_block_size(max_block_size_) {} String getName() const override { return "NonMergeJoined"; } - Block getHeader() const override { return result_sample_block; } protected: - Block readImpl() override - { - if (parent.getRightBlocksCount()) - return createBlock(); - return {}; - } - -private: - const MergeJoin & parent; - size_t max_block_size; - size_t block_number = 0; - - Block createBlock() - { - MutableColumns columns_right = saved_block_sample.cloneEmptyColumns(); - - size_t rows_added = fillColumns(columns_right); - if (!rows_added) - return {}; - - Block res = result_sample_block.cloneEmpty(); - addLeftColumns(res, rows_added); - addRightColumns(res, columns_right); - copySameKeys(res); - correctLowcardAndNullability(res); - return res; - } - - size_t fillColumns(MutableColumns & columns_right) + size_t fillColumns(MutableColumns & columns_right) override { const RowBitmaps & bitmaps = *parent.used_rows_bitmap; size_t rows_added = 0; @@ -1127,13 +1090,19 @@ private: return rows_added; } + +private: + const MergeJoin & parent; + size_t max_block_size; + size_t block_number = 0; }; BlockInputStreamPtr MergeJoin::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const { + size_t left_columns_count = result_sample_block.columns() - right_columns_to_add.columns(); if (table_join->strictness() == ASTTableJoin::Strictness::All && (is_right || is_full)) - return std::make_shared(*this, result_sample_block, key_names_right, max_block_size); + return std::make_shared(*this, result_sample_block, left_columns_count, max_block_size); return {}; } diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 844c730de4f..4aa26ead0a0 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -78,6 +78,7 @@ private: SortDescription right_merge_description; Block right_sample_block; Block right_table_keys; + /// Columns from right side of join, both key and additional Block right_columns_to_add; SortedBlocksWriter::Blocks right_blocks; diff --git a/src/Interpreters/TableJoin.cpp b/src/Interpreters/TableJoin.cpp index 20e8f6b18b4..68195dd9483 100644 --- a/src/Interpreters/TableJoin.cpp +++ b/src/Interpreters/TableJoin.cpp @@ -472,6 +472,24 @@ void TableJoin::addJoinCondition(const ASTPtr & ast, bool is_left) on_filter_condition_asts_right.push_back(ast); } +std::unordered_map TableJoin::leftToRightKeyRemap() const +{ + std::unordered_map left_to_right_key_remap; + if (hasUsing()) + { + const auto & required_right_keys = requiredRightKeys(); + for (size_t i = 0; i < key_names_left.size(); ++i) + { + const String & left_key_name = key_names_left[i]; + const String & right_key_name = key_names_right[i]; + + if (!required_right_keys.contains(right_key_name)) + left_to_right_key_remap[left_key_name] = right_key_name; + } + } + return left_to_right_key_remap; +} + /// Returns all conditions related to one table joined with 'and' function static ASTPtr buildJoinConditionColumn(const ASTs & on_filter_condition_asts) { diff --git a/src/Interpreters/TableJoin.h b/src/Interpreters/TableJoin.h index 4c8c16028f5..f0f8c3696b3 100644 --- a/src/Interpreters/TableJoin.h +++ b/src/Interpreters/TableJoin.h @@ -230,6 +230,7 @@ public: Block getRequiredRightKeys(const Block & right_table_keys, std::vector & keys_sources) const; String renamedRightColumnName(const String & name) const; + std::unordered_map leftToRightKeyRemap() const; }; } diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index e9f3e4f3fdd..2c6a2731a0e 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -314,8 +314,16 @@ void removeLowCardinalityInplace(Block & block, const Names & names, bool change } } -void restoreLowCardinalityInplace(Block & block) +void restoreLowCardinalityInplace(Block & block, const Names & lowcard_keys) { + for (const auto & column_name : lowcard_keys) + { + if (!block.has(column_name)) + continue; + if (auto & col = block.getByName(column_name); !col.type->lowCardinality()) + JoinCommon::changeLowCardinalityInplace(col); + } + for (size_t i = 0; i < block.columns(); ++i) { auto & col = block.getByPosition(i); @@ -484,49 +492,21 @@ void splitAdditionalColumns(const Names & key_names, const Block & sample_block, } - -NotJoined::NotJoined(const TableJoin & table_join, const Block & saved_block_sample_, const Block & right_sample_block, - const Block & result_sample_block_, const Names & key_names_left_, const Names & key_names_right_) +NotJoined::NotJoined(const Block & saved_block_sample_, + const Block & result_sample_block_, + size_t left_columns_count, + const LeftToRightKeyRemap & left_to_right_key_remap) : saved_block_sample(saved_block_sample_) , result_sample_block(materializeBlock(result_sample_block_)) - , key_names_left(key_names_left_.empty() ? table_join.keyNamesLeft() : key_names_left_) - , key_names_right(key_names_right_.empty() ? table_join.keyNamesRight() : key_names_right_) { - std::vector tmp; - Block right_table_keys; - Block sample_block_with_columns_to_add; - - JoinCommon::splitAdditionalColumns(key_names_right, right_sample_block, right_table_keys, - sample_block_with_columns_to_add); - Block required_right_keys = table_join.getRequiredRightKeys(right_table_keys, tmp); - - std::unordered_map left_to_right_key_remap; - - if (table_join.hasUsing()) - { - for (size_t i = 0; i < key_names_left.size(); ++i) - { - const String & left_key_name = key_names_left[i]; - const String & right_key_name = key_names_right[i]; - - size_t left_key_pos = result_sample_block.getPositionByName(left_key_name); - size_t right_key_pos = saved_block_sample.getPositionByName(right_key_name); - - if (!required_right_keys.has(right_key_name)) - left_to_right_key_remap[left_key_pos] = right_key_pos; - } - } - - /// result_sample_block: left_sample_block + left expressions, right not key columns, required right keys - size_t left_columns_count = result_sample_block.columns() - - sample_block_with_columns_to_add.columns() - required_right_keys.columns(); - for (size_t left_pos = 0; left_pos < left_columns_count; ++left_pos) { - /// We need right 'x' for 'RIGHT JOIN ... USING(x)'. - if (left_to_right_key_remap.count(left_pos)) + /// We need right 'x' for 'RIGHT JOIN ... USING(x)' + auto left_name = result_sample_block.getByPosition(left_pos).name; + const auto & right_key = left_to_right_key_remap.find(left_name); + if (right_key != left_to_right_key_remap.end()) { - size_t right_key_pos = left_to_right_key_remap[left_pos]; + size_t right_key_pos = saved_block_sample.getPositionByName(right_key->second); setRightIndex(right_key_pos, left_pos); } else @@ -558,7 +538,7 @@ NotJoined::NotJoined(const TableJoin & table_join, const Block & saved_block_sam void NotJoined::setRightIndex(size_t right_pos, size_t result_position) { - if (!column_indices_right.count(right_pos)) + if (!column_indices_right.contains(right_pos)) { column_indices_right[right_pos] = result_position; extractColumnChanges(right_pos, result_position); diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 1f037899155..f61e110627b 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -30,7 +31,7 @@ ColumnRawPtrs materializeColumnsInplace(Block & block, const Names & names); ColumnRawPtrs getRawPointers(const Columns & columns); void removeLowCardinalityInplace(Block & block); void removeLowCardinalityInplace(Block & block, const Names & names, bool change_type = true); -void restoreLowCardinalityInplace(Block & block); +void restoreLowCardinalityInplace(Block & block, const Names & lowcard_keys); ColumnRawPtrs extractKeysForJoin(const Block & block_keys, const Names & key_names_right); @@ -64,40 +65,69 @@ void changeLowCardinalityInplace(ColumnWithTypeAndName & column); } /// Creates result from right table data in RIGHT and FULL JOIN when keys are not present in left table. -class NotJoined +class NotJoined : public IBlockInputStream { public: - NotJoined(const TableJoin & table_join, const Block & saved_block_sample_, const Block & right_sample_block, - const Block & result_sample_block_, const Names & key_names_left_ = {}, const Names & key_names_right_ = {}); + using LeftToRightKeyRemap = std::unordered_map; + NotJoined(const Block & saved_block_sample_, + const Block & result_sample_block_, + size_t left_columns_count, + const LeftToRightKeyRemap & left_to_right_key_remap); + + String getName() const override { return "NonJoined"; } + Block getHeader() const override { return result_sample_block; } + +protected: + Block readImpl() override final + { + Block result = saved_block_sample.cloneEmpty(); + MutableColumns columns_right = result.mutateColumns(); + + size_t rows_added = fillColumns(columns_right); + if (rows_added == 0) + return {}; + + Block res = result_sample_block.cloneEmpty(); + addLeftColumns(res, rows_added); + addRightColumns(res, columns_right); + copySameKeys(res); + correctLowcardAndNullability(res); + +#ifndef NDEBUG + assertBlocksHaveEqualStructure(res, result_sample_block, getName()); +#endif + return res; + } + + virtual size_t fillColumns(MutableColumns & columns_right) = 0; + +private: + void extractColumnChanges(size_t right_pos, size_t result_pos); void correctLowcardAndNullability(Block & block); void addLeftColumns(Block & block, size_t rows_added) const; void addRightColumns(Block & block, MutableColumns & columns_right) const; void copySameKeys(Block & block) const; -protected: + /// Right block saved in Join Block saved_block_sample; + + /// Output of join Block result_sample_block; - Names key_names_left; - Names key_names_right; - - ~NotJoined() = default; - -private: /// Indices of columns in result_sample_block that should be generated std::vector column_indices_left; /// Indices of columns that come from the right-side table: right_pos -> result_pos std::unordered_map column_indices_right; - /// + std::unordered_map same_result_keys; - /// Which right columns (saved in parent) need nullability change before placing them in result block + + /// Which right columns (saved in parent) need Nullability/LowCardinality change + /// before placing them in result block std::vector> right_nullability_changes; - /// Which right columns (saved in parent) need LowCardinality change before placing them in result block std::vector> right_lowcard_changes; void setRightIndex(size_t right_pos, size_t result_position); - void extractColumnChanges(size_t right_pos, size_t result_pos); }; } diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index e402fd788bc..c1329d02fed 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -1,7 +1,6 @@ #include #include #include -#include #include From 9d8178d04c6321ad301ee82ead42106a2bb928f9 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 9 Aug 2021 17:30:37 +0300 Subject: [PATCH 125/220] Refactor NotJoined pt2: rename classes, get rig of inheritance --- src/Interpreters/HashJoin.cpp | 21 +++++++--------- src/Interpreters/HashJoin.h | 2 +- src/Interpreters/MergeJoin.cpp | 24 +++++++------------ src/Interpreters/MergeJoin.h | 2 +- src/Interpreters/join_common.cpp | 37 +++++++++++++++++++++------- src/Interpreters/join_common.h | 41 +++++++++++++------------------- 6 files changed, 65 insertions(+), 62 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 4130431485e..4384072377d 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -629,7 +629,7 @@ bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits) ConstNullMapPtr null_map{}; ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map); - /// If RIGHT or FULL save blocks with nulls for NonJoinedBlockInputStream + /// If RIGHT or FULL save blocks with nulls for NotJoinedInputStream UInt8 save_nullmap = 0; if (isRightOrFull(kind) && null_map) { @@ -1468,21 +1468,15 @@ struct AdderNonJoined /// Stream from not joined earlier rows of the right table. -class NonJoinedBlockInputStream final : public NotJoined +class NotJoinedHash final : public NotJoinedInputStream::RightColumnsFiller { public: - NonJoinedBlockInputStream( - const HashJoin & parent_, - const Block & result_sample_block_, - size_t left_columns_count, - UInt64 max_block_size_) - : NotJoined(parent_.savedBlockSample(), result_sample_block_, - left_columns_count, parent_.table_join->leftToRightKeyRemap()) - , parent(parent_) - , max_block_size(max_block_size_) + NotJoinedHash(const HashJoin & parent_, UInt64 max_block_size_) + : parent(parent_), max_block_size(max_block_size_) {} -protected: + Block getEmptyBlock() override { return parent.savedBlockSample().cloneEmpty(); } + size_t fillColumns(MutableColumns & columns_right) override { size_t rows_added = 0; @@ -1594,7 +1588,8 @@ BlockInputStreamPtr HashJoin::createStreamWithNonJoinedRows(const Block & result } size_t left_columns_count = result_sample_block.columns() - required_right_keys.columns() - sample_block_with_columns_to_add.columns(); - return std::make_shared(*this, result_sample_block, left_columns_count, max_block_size); + auto non_joined = std::make_unique(*this, max_block_size); + return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); } void HashJoin::reuseJoinedData(const HashJoin & join) diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index 65e3f5dbabe..f6e47b59d25 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -337,7 +337,7 @@ public: bool isUsed(size_t off) const { return used_flags.getUsedSafe(off); } private: - friend class NonJoinedBlockInputStream; + friend class NotJoinedHash; friend class JoinSource; std::shared_ptr table_join; diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 84d5a80cff5..611f1742fa4 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -1028,24 +1028,15 @@ void MergeJoin::initRightTableWriter() } /// Stream from not joined earlier rows of the right table. -class NonMergeJoinedBlockInputStream final : public NotJoined +class NotJoinedMerge final : public NotJoinedInputStream::RightColumnsFiller { public: - NonMergeJoinedBlockInputStream(const MergeJoin & parent_, - const Block & result_sample_block, - size_t left_columns_count, - UInt64 max_block_size_) - : NotJoined(parent_.modifyRightBlock(parent_.right_sample_block), - result_sample_block, - left_columns_count, - parent_.table_join->leftToRightKeyRemap()) - , parent(parent_) - , max_block_size(max_block_size_) + NotJoinedMerge(const MergeJoin & parent_, UInt64 max_block_size_) + : parent(parent_), max_block_size(max_block_size_) {} - String getName() const override { return "NonMergeJoined"; } + Block getEmptyBlock() override { return parent.modifyRightBlock(parent.right_sample_block).cloneEmpty(); } -protected: size_t fillColumns(MutableColumns & columns_right) override { const RowBitmaps & bitmaps = *parent.used_rows_bitmap; @@ -1100,9 +1091,12 @@ private: BlockInputStreamPtr MergeJoin::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const { - size_t left_columns_count = result_sample_block.columns() - right_columns_to_add.columns(); if (table_join->strictness() == ASTTableJoin::Strictness::All && (is_right || is_full)) - return std::make_shared(*this, result_sample_block, left_columns_count, max_block_size); + { + size_t left_columns_count = result_sample_block.columns() - right_columns_to_add.columns(); + auto non_joined = std::make_unique(*this, max_block_size); + return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); + } return {}; } diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 4aa26ead0a0..89371d8b13b 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -38,7 +38,7 @@ public: BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const override; private: - friend class NonMergeJoinedBlockInputStream; + friend class NotJoinedMerge; struct NotProcessed : public ExtraBlock { diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 2c6a2731a0e..c640fea3a36 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -492,11 +492,12 @@ void splitAdditionalColumns(const Names & key_names, const Block & sample_block, } -NotJoined::NotJoined(const Block & saved_block_sample_, +NotJoinedInputStream::NotJoinedInputStream(std::unique_ptr filler_, const Block & result_sample_block_, size_t left_columns_count, const LeftToRightKeyRemap & left_to_right_key_remap) - : saved_block_sample(saved_block_sample_) + : filler(std::move(filler_)) + , saved_block_sample(filler->getEmptyBlock()) , result_sample_block(materializeBlock(result_sample_block_)) { for (size_t left_pos = 0; left_pos < left_columns_count; ++left_pos) @@ -536,7 +537,7 @@ NotJoined::NotJoined(const Block & saved_block_sample_, ErrorCodes::LOGICAL_ERROR); } -void NotJoined::setRightIndex(size_t right_pos, size_t result_position) +void NotJoinedInputStream::setRightIndex(size_t right_pos, size_t result_position) { if (!column_indices_right.contains(right_pos)) { @@ -547,7 +548,7 @@ void NotJoined::setRightIndex(size_t right_pos, size_t result_position) same_result_keys[result_position] = column_indices_right[right_pos]; } -void NotJoined::extractColumnChanges(size_t right_pos, size_t result_pos) +void NotJoinedInputStream::extractColumnChanges(size_t right_pos, size_t result_pos) { auto src_props = getLowcardAndNullability(saved_block_sample.getByPosition(right_pos).column); auto dst_props = getLowcardAndNullability(result_sample_block.getByPosition(result_pos).column); @@ -559,7 +560,7 @@ void NotJoined::extractColumnChanges(size_t right_pos, size_t result_pos) right_lowcard_changes.push_back({result_pos, dst_props.is_lowcard}); } -void NotJoined::correctLowcardAndNullability(Block & block) +void NotJoinedInputStream::correctLowcardAndNullability(Block & block) { for (auto & [pos, added] : right_nullability_changes) { @@ -587,7 +588,7 @@ void NotJoined::correctLowcardAndNullability(Block & block) } } -void NotJoined::addLeftColumns(Block & block, size_t rows_added) const +void NotJoinedInputStream::addLeftColumns(Block & block, size_t rows_added) const { for (size_t pos : column_indices_left) { @@ -599,7 +600,7 @@ void NotJoined::addLeftColumns(Block & block, size_t rows_added) const } } -void NotJoined::addRightColumns(Block & block, MutableColumns & columns_right) const +void NotJoinedInputStream::addRightColumns(Block & block, MutableColumns & columns_right) const { for (const auto & pr : column_indices_right) { @@ -609,7 +610,7 @@ void NotJoined::addRightColumns(Block & block, MutableColumns & columns_right) c } } -void NotJoined::copySameKeys(Block & block) const +void NotJoinedInputStream::copySameKeys(Block & block) const { for (const auto & pr : same_result_keys) { @@ -619,4 +620,24 @@ void NotJoined::copySameKeys(Block & block) const } } +Block NotJoinedInputStream::readImpl() + +{ + Block right_block = filler->getEmptyBlock(); + MutableColumns columns_right = right_block.cloneEmptyColumns(); + size_t rows_added = filler->fillColumns(columns_right); + if (rows_added == 0) + return {}; + + addLeftColumns(right_block, rows_added); + addRightColumns(right_block, columns_right); + copySameKeys(right_block); + correctLowcardAndNullability(right_block); + +#ifndef NDEBUG + assertBlocksHaveEqualStructure(right_block, result_sample_block, getName()); +#endif + return right_block; +} + } diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index f61e110627b..32fa4a4ee9e 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -65,12 +65,24 @@ void changeLowCardinalityInplace(ColumnWithTypeAndName & column); } /// Creates result from right table data in RIGHT and FULL JOIN when keys are not present in left table. -class NotJoined : public IBlockInputStream +class NotJoinedInputStream : public IBlockInputStream { public: using LeftToRightKeyRemap = std::unordered_map; - NotJoined(const Block & saved_block_sample_, + /// Returns non joined columns from right part of join + class RightColumnsFiller + { + public: + /// Create empty block for right part + virtual Block getEmptyBlock() = 0; + /// Fill columns from right part of join with not joined rows + virtual size_t fillColumns(MutableColumns & columns_right) = 0; + + virtual ~RightColumnsFiller() = default; + }; + + NotJoinedInputStream(std::unique_ptr filler_, const Block & result_sample_block_, size_t left_columns_count, const LeftToRightKeyRemap & left_to_right_key_remap); @@ -79,28 +91,7 @@ public: Block getHeader() const override { return result_sample_block; } protected: - Block readImpl() override final - { - Block result = saved_block_sample.cloneEmpty(); - MutableColumns columns_right = result.mutateColumns(); - - size_t rows_added = fillColumns(columns_right); - if (rows_added == 0) - return {}; - - Block res = result_sample_block.cloneEmpty(); - addLeftColumns(res, rows_added); - addRightColumns(res, columns_right); - copySameKeys(res); - correctLowcardAndNullability(res); - -#ifndef NDEBUG - assertBlocksHaveEqualStructure(res, result_sample_block, getName()); -#endif - return res; - } - - virtual size_t fillColumns(MutableColumns & columns_right) = 0; + Block readImpl() override final; private: void extractColumnChanges(size_t right_pos, size_t result_pos); @@ -109,6 +100,8 @@ private: void addRightColumns(Block & block, MutableColumns & columns_right) const; void copySameKeys(Block & block) const; + std::unique_ptr filler; + /// Right block saved in Join Block saved_block_sample; From e8e650b16b2afeed6a744294d0c4a1d0aecb045a Mon Sep 17 00:00:00 2001 From: Artur <613623@mail.ru> Date: Tue, 17 Aug 2021 12:01:31 +0000 Subject: [PATCH 126/220] correct infile form local --- .../getSourceFromFromASTInsertQuery.cpp | 39 +++++++++++++------ .../0_stateless/02009_from_infile.reference | 2 +- .../queries/0_stateless/02009_from_infile.sh | 2 +- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp index eb2c1b91cba..75750211907 100644 --- a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -11,6 +12,8 @@ #include #include #include +#include "IO/CompressionMethod.h" +#include "Parsers/ASTLiteral.h" namespace DB @@ -36,7 +39,7 @@ Pipe getSourceFromFromASTInsertQuery( if (!ast_insert_query) throw Exception("Logical error: query requires data to insert, but it is not INSERT query", ErrorCodes::LOGICAL_ERROR); - if (ast_insert_query->infile) + if (ast_insert_query->infile && context->getApplicationType() == Context::ApplicationType::SERVER) throw Exception("Query has infile and was send directly to server", ErrorCodes::UNKNOWN_TYPE_OF_QUERY); String format = ast_insert_query->format; @@ -52,20 +55,32 @@ Pipe getSourceFromFromASTInsertQuery( auto input_buffer_ast_part = std::make_unique( ast_insert_query->data, ast_insert_query->data ? ast_insert_query->end - ast_insert_query->data : 0); - ConcatReadBuffer::ReadBuffers buffers; - if (ast_insert_query->data) - buffers.push_back(input_buffer_ast_part.get()); + std::unique_ptr input_buffer; - if (input_buffer_tail_part) - buffers.push_back(input_buffer_tail_part); + if (ast_insert_query->infile) + { + /// Data can be from infile + const auto & in_file_node = ast_insert_query->infile->as(); + const auto in_file = in_file_node.value.safeGet(); - /** NOTE Must not read from 'input_buffer_tail_part' before read all between 'ast_insert_query.data' and 'ast_insert_query.end'. - * - because 'query.data' could refer to memory piece, used as buffer for 'input_buffer_tail_part'. - */ + input_buffer = wrapReadBufferWithCompressionMethod(std::make_unique(in_file), chooseCompressionMethod(in_file, "")); + } else + { + ConcatReadBuffer::ReadBuffers buffers; + if (ast_insert_query->data) + buffers.push_back(input_buffer_ast_part.get()); - auto input_buffer_contacenated = std::make_unique(buffers); + if (input_buffer_tail_part) + buffers.push_back(input_buffer_tail_part); - auto source = FormatFactory::instance().getInput(format, *input_buffer_contacenated, header, context, context->getSettings().max_insert_block_size); + /** NOTE Must not read from 'input_buffer_tail_part' before read all between 'ast_insert_query.data' and 'ast_insert_query.end'. + * - because 'query.data' could refer to memory piece, used as buffer for 'input_buffer_tail_part'. + */ + + input_buffer = std::make_unique(buffers); + } + + auto source = FormatFactory::instance().getInput(format, *input_buffer, header, context, context->getSettings().max_insert_block_size); Pipe pipe(source); if (context->getSettingsRef().input_format_defaults_for_omitted_fields && ast_insert_query->table_id && !input_function) @@ -83,7 +98,7 @@ Pipe getSourceFromFromASTInsertQuery( } source->addBuffer(std::move(input_buffer_ast_part)); - source->addBuffer(std::move(input_buffer_contacenated)); + source->addBuffer(std::move(input_buffer)); return pipe; } diff --git a/tests/queries/0_stateless/02009_from_infile.reference b/tests/queries/0_stateless/02009_from_infile.reference index bfad8971fe4..48483fe50c1 100644 --- a/tests/queries/0_stateless/02009_from_infile.reference +++ b/tests/queries/0_stateless/02009_from_infile.reference @@ -1,3 +1,3 @@ Hello -Correct Local +Hello Correct URL diff --git a/tests/queries/0_stateless/02009_from_infile.sh b/tests/queries/0_stateless/02009_from_infile.sh index d50e22d3e6d..578ac14f558 100755 --- a/tests/queries/0_stateless/02009_from_infile.sh +++ b/tests/queries/0_stateless/02009_from_infile.sh @@ -19,7 +19,7 @@ ${CLICKHOUSE_CLIENT} --query "INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_ ${CLICKHOUSE_CLIENT} --query "SELECT * FROM test_infile;" # if it not fails, select will print information -${CLICKHOUSE_LOCAL} --query "CREATE TABLE test_infile (word String) ENGINE=Memory(); INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV; SELECT * from test_infile;" 2>&1 | grep -q "UNKNOWN_TYPE_OF_QUERY" && echo "Correct Local" || echo 'Fail' +${CLICKHOUSE_LOCAL} --query "CREATE TABLE test_infile (word String) ENGINE=Memory(); INSERT INTO test_infile FROM INFILE '${CLICKHOUSE_TMP}/test_infile.gz' FORMAT CSV; SELECT * from test_infile;" ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=DROP+TABLE" -d 'IF EXISTS test_infile_url' ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&query=CREATE" -d 'TABLE test_infile_url (x String) ENGINE = Memory' From 67e7cc1dd5c5e8ee540d27157a290ac27496278c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 17 Aug 2021 15:21:42 +0300 Subject: [PATCH 127/220] fix some tests --- ...rrent_recreate_reattach_and_show_tables_long.sh | 14 ++++++-------- .../01294_system_distributed_on_cluster.sql | 2 -- .../01319_manual_write_to_replicas_long.sql | 6 ++++-- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh index f5a4a1adac0..6b89e7a5a8a 100755 --- a/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh +++ b/tests/queries/0_stateless/01294_lazy_database_concurrent_recreate_reattach_and_show_tables_long.sh @@ -101,14 +101,12 @@ timeout $TIMEOUT bash -c test_func 2> /dev/null & wait sleep 1 -${CLICKHOUSE_CLIENT} -n -q " - DROP TABLE IF EXISTS $CURR_DATABASE.log; - DROP TABLE IF EXISTS $CURR_DATABASE.slog; - DROP TABLE IF EXISTS $CURR_DATABASE.tlog; - DROP TABLE IF EXISTS $CURR_DATABASE.tlog2; -" +for table in log tlog slog tlog2; do + $CLICKHOUSE_CLIENT -q "SYSTEM STOP TTL MERGES $CURR_DATABASE.$table" >& /dev/null + ${CLICKHOUSE_CLIENT} -q "ATTACH TABLE $CURR_DATABASE.$table;" 2>/dev/null +done + +${CLICKHOUSE_CLIENT} -q "DROP DATABASE $CURR_DATABASE" -$CLICKHOUSE_CLIENT -q "SYSTEM START TTL MERGES"; echo "Test OK" -# TODO: doesn't work! $CLICKHOUSE_CLIENT -q "DROP DATABASE $CURR_DATABASE" diff --git a/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql b/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql index 525974e78ba..e5cd3ea362f 100644 --- a/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql +++ b/tests/queries/0_stateless/01294_system_distributed_on_cluster.sql @@ -11,11 +11,9 @@ create table db_01294.dist_01294 as system.one engine=Distributed(test_shard_loc system flush distributed db_01294.dist_01294; system flush distributed on cluster test_shard_localhost db_01294.dist_01294; -- stop -system stop distributed sends; system stop distributed sends db_01294.dist_01294; system stop distributed sends on cluster test_shard_localhost db_01294.dist_01294; -- start -system start distributed sends; system start distributed sends db_01294.dist_01294; system start distributed sends on cluster test_shard_localhost db_01294.dist_01294; diff --git a/tests/queries/0_stateless/01319_manual_write_to_replicas_long.sql b/tests/queries/0_stateless/01319_manual_write_to_replicas_long.sql index 7fb4d0b7d61..cec75ac1b23 100644 --- a/tests/queries/0_stateless/01319_manual_write_to_replicas_long.sql +++ b/tests/queries/0_stateless/01319_manual_write_to_replicas_long.sql @@ -4,7 +4,8 @@ DROP TABLE IF EXISTS r2; CREATE TABLE r1 (x String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/r', 'r1') ORDER BY x; CREATE TABLE r2 (x String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/r', 'r2') ORDER BY x; -SYSTEM STOP REPLICATED SENDS; +SYSTEM STOP REPLICATED SENDS r1; +SYSTEM STOP REPLICATED SENDS r2; INSERT INTO r1 VALUES ('Hello, world'); SELECT * FROM r1; @@ -14,7 +15,8 @@ SELECT '---'; SELECT * FROM r1; SELECT * FROM r2; -SYSTEM START REPLICATED SENDS; +SYSTEM START REPLICATED SENDS r1; +SYSTEM START REPLICATED SENDS r2; SYSTEM SYNC REPLICA r1; SYSTEM SYNC REPLICA r2; From d8778098e723141791bc15169db2157474733dc6 Mon Sep 17 00:00:00 2001 From: Nikita Mikhalev Date: Tue, 17 Aug 2021 17:22:30 +0500 Subject: [PATCH 128/220] =?UTF-8?q?=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=BB?= =?UTF-8?q?=D1=8F=D0=B5=D1=82=20=D0=BF=D1=80=D0=B5=D0=B4=D0=BE=D1=81=D1=82?= =?UTF-8?q?=D0=B5=D1=80=D0=B5=D0=B6=D0=B5=D0=BD=D0=B8=D0=B5=20=D0=BE=20?= =?UTF-8?q?=D0=B2=D1=8B=D0=BF=D0=BE=D0=BB=D0=BD=D0=B5=D0=BD=D0=B8=D0=B8=20?= =?UTF-8?q?ALTER=20=D0=BD=D0=B0=20=D1=82=D0=B0=D0=B1=D0=BB=D0=B8=D1=86?= =?UTF-8?q?=D0=B5=20Buffer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/ru/engines/table-engines/special/buffer.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/special/buffer.md b/docs/ru/engines/table-engines/special/buffer.md index ba865b72b78..3e3c9226933 100644 --- a/docs/ru/engines/table-engines/special/buffer.md +++ b/docs/ru/engines/table-engines/special/buffer.md @@ -48,7 +48,10 @@ CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10 Если у одного из столбцов таблицы Buffer и подчинённой таблицы не совпадает тип, то в лог сервера будет записано сообщение об ошибке и буфер будет очищен. То же самое происходит, если подчинённая таблица не существует в момент сброса буфера. -Если есть необходимость выполнить ALTER для подчинённой таблицы и для таблицы Buffer, то рекомендуется удалить таблицу Buffer, затем выполнить ALTER подчинённой таблицы, а затем создать таблицу Buffer заново. +Если есть необходимость выполнить ALTER для подчинённой таблицы и для таблицы Buffer, то рекомендуется удалить таблицу Buffer, затем выполнить ALTER подчинённой таблицы, а затем создать таблицу Buffer заново. + +!!! attention "Внимание" + В релизах до 28 сентября 2020 года выполнение ALTER на таблице Buffer ломает структуру блоков и вызывает ошибку (см. [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), поэтому удаление буфера и его создание заново - единственный вариант миграции для данного движка. Убедитесь, что в вашей версии ошибка устранена перед выполнением ALTER на таблице Buffer. При нештатном перезапуске сервера, данные, находящиеся в буфере, будут потеряны. From 189ad46206a50abcff14250b84f93ae3bbff0bb2 Mon Sep 17 00:00:00 2001 From: Nikita Mikhalev Date: Tue, 17 Aug 2021 17:30:05 +0500 Subject: [PATCH 129/220] Added attention for ALTER on Buffer table prior to 28.10.2020 --- docs/en/engines/table-engines/special/buffer.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index cacb310a15c..a31bb462754 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -56,6 +56,9 @@ The same thing happens if the subordinate table does not exist when the buffer i If you need to run ALTER for a subordinate table, and the Buffer table, we recommend first deleting the Buffer table, running ALTER for the subordinate table, then creating the Buffer table again. +!!! attention "Attention" + Running ALTER on Buffer table in releases made prior to 28 Sep 2020 will cause `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), so deleting Buffer table and then recreating it is the only option. Please check error is fixed in your release before trying to run ALTER on Buffer table. + If the server is restarted abnormally, the data in the buffer is lost. `FINAL` and `SAMPLE` do not work correctly for Buffer tables. These conditions are passed to the destination table, but are not used for processing data in the buffer. If these features are required we recommend only using the Buffer table for writing, while reading from the destination table. From 4d71f650825043019d17cd01ff170adebd0bea6c Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Tue, 17 Aug 2021 16:24:14 +0300 Subject: [PATCH 130/220] fix build --- base/daemon/BaseDaemon.cpp | 47 ------------------------------ base/daemon/BaseDaemon.h | 7 ----- base/daemon/SentryWriter.cpp | 4 +-- programs/keeper/Keeper.cpp | 3 +- programs/server/Server.cpp | 3 +- src/Common/ServerUUID.cpp | 56 ++++++++++++++++++++++++++++++++++++ src/Common/ServerUUID.h | 26 +++++++++++++++++ src/Common/getServerUUID.cpp | 12 -------- src/Common/getServerUUID.h | 5 ---- src/Functions/serverUUID.cpp | 4 +-- 10 files changed, 90 insertions(+), 77 deletions(-) create mode 100644 src/Common/ServerUUID.cpp create mode 100644 src/Common/ServerUUID.h delete mode 100644 src/Common/getServerUUID.cpp delete mode 100644 src/Common/getServerUUID.h diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index 060c812590b..745e020c8bb 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -44,7 +44,6 @@ #include #include #include -#include #include #include #include @@ -1060,49 +1059,3 @@ String BaseDaemon::getStoredBinaryHash() const { return stored_binary_hash; } - -void BaseDaemon::loadServerUUID(const fs::path & server_uuid_file, Poco::Logger * log) -{ - /// Write a uuid file containing a unique uuid if the file doesn't already exist during server start. - - if (fs::exists(server_uuid_file)) - { - try - { - DB::UUID uuid; - DB::ReadBufferFromFile in(server_uuid_file); - DB::readUUIDText(uuid, in); - DB::assertEOF(in); - server_uuid = uuid; - return; - } - catch (...) - { - /// As for now it's ok to just overwrite it, because persistency in not essential. - LOG_ERROR(log, "Cannot read server UUID from file {}: {}. Will overwrite it", - server_uuid_file.string(), DB::getCurrentExceptionMessage(true)); - } - } - - try - { - DB::UUID new_uuid = DB::UUIDHelpers::generateV4(); - auto uuid_str = DB::toString(new_uuid); - DB::WriteBufferFromFile out(server_uuid_file); - out.write(uuid_str.data(), uuid_str.size()); - out.sync(); - out.finalize(); - server_uuid = new_uuid; - } - catch (...) - { - throw Poco::Exception( - "Caught Exception " + DB::getCurrentExceptionMessage(true) + " while writing the Server UUID file " - + server_uuid_file.string()); - } -} - -DB::UUID BaseDaemon::getServerUUID() const -{ - return server_uuid; -} diff --git a/base/daemon/BaseDaemon.h b/base/daemon/BaseDaemon.h index 65c25ae0d57..3d47d85a9f5 100644 --- a/base/daemon/BaseDaemon.h +++ b/base/daemon/BaseDaemon.h @@ -5,7 +5,6 @@ #include #include #include -#include #include #include #include @@ -25,7 +24,6 @@ #include #include -namespace fs = std::filesystem; /// \brief Base class for applications that can run as daemons. /// @@ -126,9 +124,6 @@ public: /// Hash of the binary for integrity checks. String getStoredBinaryHash() const; - void loadServerUUID(const fs::path & server_uuid_file, Poco::Logger * log); - DB::UUID getServerUUID() const; - protected: virtual void logRevision() const; @@ -184,8 +179,6 @@ protected: bool should_setup_watchdog = false; char * argv0 = nullptr; - - DB::UUID server_uuid = DB::UUIDHelpers::Nil; }; diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index 7578f93f5ed..de772afdec3 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #if !defined(ARCADIA_BUILD) # include "Common/config_version.h" @@ -39,7 +39,7 @@ void setExtras() if (!anonymize) sentry_set_extra("server_name", sentry_value_new_string(getFQDNOrHostName().c_str())); - DB::UUID server_uuid = getServerUUID(); + DB::UUID server_uuid = DB::ServerUUID::get(); if (server_uuid != DB::UUIDHelpers::Nil) { std::string server_uuid_str = DB::toString(server_uuid); diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index fd225247795..c35e3e64d37 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include @@ -326,7 +327,7 @@ int Keeper::main(const std::vector & /*args*/) } } - loadServerUUID(path + "/uuid", log); + DB::ServerUUID::load(path + "/uuid", log); const Settings & settings = global_context->getSettingsRef(); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 8685e21ccb4..7e2c250d6e5 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -695,7 +696,7 @@ if (ThreadFuzzer::instance().isEffective()) StatusFile status{path / "status", StatusFile::write_full_info}; - loadServerUUID(path / "uuid", log); + DB::ServerUUID::load(path / "uuid", log); /// Try to increase limit on number of open files. { diff --git a/src/Common/ServerUUID.cpp b/src/Common/ServerUUID.cpp new file mode 100644 index 00000000000..486b0206e56 --- /dev/null +++ b/src/Common/ServerUUID.cpp @@ -0,0 +1,56 @@ +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int CANNOT_CREATE_FILE; +} + +void ServerUUID::load(const fs::path & server_uuid_file, Poco::Logger * log) +{ + /// Write a uuid file containing a unique uuid if the file doesn't already exist during server start. + + if (fs::exists(server_uuid_file)) + { + try + { + UUID uuid; + ReadBufferFromFile in(server_uuid_file); + readUUIDText(uuid, in); + assertEOF(in); + server_uuid = uuid; + return; + } + catch (...) + { + /// As for now it's ok to just overwrite it, because persistency in not essential. + LOG_ERROR(log, "Cannot read server UUID from file {}: {}. Will overwrite it", + server_uuid_file.string(), getCurrentExceptionMessage(true)); + } + } + + try + { + UUID new_uuid = UUIDHelpers::generateV4(); + auto uuid_str = toString(new_uuid); + WriteBufferFromFile out(server_uuid_file); + out.write(uuid_str.data(), uuid_str.size()); + out.sync(); + out.finalize(); + server_uuid = new_uuid; + } + catch (...) + { + throw Exception(ErrorCodes::CANNOT_CREATE_FILE, "Caught Exception {} while writing the Server UUID file {}", + getCurrentExceptionMessage(false), server_uuid_file.string()); + } +} + +} diff --git a/src/Common/ServerUUID.h b/src/Common/ServerUUID.h new file mode 100644 index 00000000000..36bbf0e6315 --- /dev/null +++ b/src/Common/ServerUUID.h @@ -0,0 +1,26 @@ +#pragma once +#include +#include + +namespace fs = std::filesystem; +namespace Poco +{ + class Logger; +} + +namespace DB +{ + +class ServerUUID +{ + inline static UUID server_uuid = UUIDHelpers::Nil; + +public: + /// Returns persistent UUID of current clickhouse-server or clickhouse-keeper instance. + static UUID get() { return server_uuid; } + + /// Loads server UUID from file or creates new one. Should be called on daemon startup. + static void load(const fs::path & server_uuid_file, Poco::Logger * log); +}; + +} diff --git a/src/Common/getServerUUID.cpp b/src/Common/getServerUUID.cpp deleted file mode 100644 index 5067bd20c29..00000000000 --- a/src/Common/getServerUUID.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include -#include -#include - -DB::UUID getServerUUID() -{ - const auto * daemon = dynamic_cast(&Poco::Util::Application::instance()); - if (daemon) - return daemon->getServerUUID(); - else - return DB::UUIDHelpers::Nil; -} diff --git a/src/Common/getServerUUID.h b/src/Common/getServerUUID.h deleted file mode 100644 index 107dff51f5c..00000000000 --- a/src/Common/getServerUUID.h +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once -#include - -/// Returns persistent UUID of current clickhouse-server or clickhouse-keeper instance. -DB::UUID getServerUUID(); diff --git a/src/Functions/serverUUID.cpp b/src/Functions/serverUUID.cpp index 5d076ba2a20..d896d56e21a 100644 --- a/src/Functions/serverUUID.cpp +++ b/src/Functions/serverUUID.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -17,7 +17,7 @@ class FunctionServerUUID : public IFunction static FunctionPtr create(ContextPtr context) { - return std::make_shared(context->isDistributed(), getServerUUID()); + return std::make_shared(context->isDistributed(), ServerUUID::get()); } explicit FunctionServerUUID(bool is_distributed_, UUID server_uuid_) From bc853d250f8e73859a5ad5f7299e0b68e291cf90 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Tue, 17 Aug 2021 16:24:28 +0300 Subject: [PATCH 131/220] Set function divide as suitable for short-circuit in case of Nullable(Decimal) --- src/Functions/FunctionBinaryArithmetic.h | 2 +- src/Functions/FunctionHelpers.cpp | 10 ++++++++++ src/Functions/FunctionHelpers.h | 2 ++ .../queries/0_stateless/01822_short_circuit.reference | 8 ++++++++ tests/queries/0_stateless/01822_short_circuit.sql | 5 +++++ 5 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/Functions/FunctionBinaryArithmetic.h b/src/Functions/FunctionBinaryArithmetic.h index 2c866d3c31c..4907bf6abda 100644 --- a/src/Functions/FunctionBinaryArithmetic.h +++ b/src/Functions/FunctionBinaryArithmetic.h @@ -958,7 +958,7 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & arguments) const override { return ((IsOperation::div_int || IsOperation::modulo) && !arguments[1].is_const) - || (IsOperation::div_floating && (isDecimal(arguments[0].type) || isDecimal(arguments[1].type))); + || (IsOperation::div_floating && (isDecimalOrNullableDecimal(arguments[0].type) || isDecimalOrNullableDecimal(arguments[1].type))); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override diff --git a/src/Functions/FunctionHelpers.cpp b/src/Functions/FunctionHelpers.cpp index eac1a7ad1a1..16dd34d0162 100644 --- a/src/Functions/FunctionHelpers.cpp +++ b/src/Functions/FunctionHelpers.cpp @@ -303,4 +303,14 @@ NullPresence getNullPresense(const ColumnsWithTypeAndName & args) return res; } +bool isDecimalOrNullableDecimal(const DataTypePtr & type) +{ + WhichDataType which(type); + if (which.isDecimal()) + return true; + if (!which.isNullable()) + return false; + return isDecimal(assert_cast(type.get())->getNestedType()); +} + } diff --git a/src/Functions/FunctionHelpers.h b/src/Functions/FunctionHelpers.h index 5abe24f4e50..5fc8a06681a 100644 --- a/src/Functions/FunctionHelpers.h +++ b/src/Functions/FunctionHelpers.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -171,4 +172,5 @@ struct NullPresence NullPresence getNullPresense(const ColumnsWithTypeAndName & args); +bool isDecimalOrNullableDecimal(const DataTypePtr & type); } diff --git a/tests/queries/0_stateless/01822_short_circuit.reference b/tests/queries/0_stateless/01822_short_circuit.reference index 204bcd0538e..101d4e1be45 100644 --- a/tests/queries/0_stateless/01822_short_circuit.reference +++ b/tests/queries/0_stateless/01822_short_circuit.reference @@ -1395,3 +1395,11 @@ Decimal32 21.00 14.00 10.50 +0.00 +42.00 +21.00 +14.00 +10.50 +\N +\N +\N diff --git a/tests/queries/0_stateless/01822_short_circuit.sql b/tests/queries/0_stateless/01822_short_circuit.sql index 16908642c52..db50721a468 100644 --- a/tests/queries/0_stateless/01822_short_circuit.sql +++ b/tests/queries/0_stateless/01822_short_circuit.sql @@ -123,3 +123,8 @@ select if(number > 0, intDiv(42, number), 1) from numbers(5); select if(number > 0, intDiv(42, number), 1) from numbers(5); select if(number > 0, 42 / toDecimal32(number, 2), 0) from numbers(5); +select if(number = 0, 0, toDecimal32(42, 2) / number) from numbers(5); +select if(isNull(x), Null, 42 / x) from (select CAST(materialize(Null), 'Nullable(Decimal32(2))') as x); +select if(isNull(x), Null, x / 0) from (select CAST(materialize(Null), 'Nullable(Decimal32(2))') as x); + +select if(isNull(x), Null, intDiv(42, x)) from (select CAST(materialize(Null), 'Nullable(Int64)') as x); From 2dfbbe364b357699e12888093540e1b6431a8e7a Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 17 Aug 2021 16:30:01 +0300 Subject: [PATCH 132/220] Do not use BlockInputStream for NonJoined --- src/Interpreters/HashJoin.cpp | 9 ++--- src/Interpreters/HashJoin.h | 3 +- src/Interpreters/IJoin.h | 4 +- src/Interpreters/JoinSwitcher.h | 38 +------------------ src/Interpreters/MergeJoin.cpp | 8 ++-- src/Interpreters/MergeJoin.h | 2 +- src/Interpreters/join_common.cpp | 18 ++++----- src/Interpreters/join_common.h | 11 ++---- .../Transforms/JoiningTransform.cpp | 8 ++-- src/Processors/Transforms/JoiningTransform.h | 5 +-- 10 files changed, 32 insertions(+), 74 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 4384072377d..6abaddd6270 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -21,7 +21,6 @@ #include -#include #include #include @@ -629,7 +628,7 @@ bool HashJoin::addJoinedBlock(const Block & source_block, bool check_limits) ConstNullMapPtr null_map{}; ColumnPtr null_map_holder = extractNestedColumnsAndNullMap(key_columns, null_map); - /// If RIGHT or FULL save blocks with nulls for NotJoinedInputStream + /// If RIGHT or FULL save blocks with nulls for NotJoinedBlocks UInt8 save_nullmap = 0; if (isRightOrFull(kind) && null_map) { @@ -1468,7 +1467,7 @@ struct AdderNonJoined /// Stream from not joined earlier rows of the right table. -class NotJoinedHash final : public NotJoinedInputStream::RightColumnsFiller +class NotJoinedHash final : public NotJoinedBlocks::RightColumnsFiller { public: NotJoinedHash(const HashJoin & parent_, UInt64 max_block_size_) @@ -1578,7 +1577,7 @@ private: }; -BlockInputStreamPtr HashJoin::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const +std::shared_ptr HashJoin::getNonJoinedBlocks(const Block & result_sample_block, UInt64 max_block_size) const { if (table_join->strictness() == ASTTableJoin::Strictness::Asof || table_join->strictness() == ASTTableJoin::Strictness::Semi || @@ -1589,7 +1588,7 @@ BlockInputStreamPtr HashJoin::createStreamWithNonJoinedRows(const Block & result size_t left_columns_count = result_sample_block.columns() - required_right_keys.columns() - sample_block_with_columns_to_add.columns(); auto non_joined = std::make_unique(*this, max_block_size); - return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); + return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); } void HashJoin::reuseJoinedData(const HashJoin & join) diff --git a/src/Interpreters/HashJoin.h b/src/Interpreters/HashJoin.h index f6e47b59d25..2e691f189c4 100644 --- a/src/Interpreters/HashJoin.h +++ b/src/Interpreters/HashJoin.h @@ -20,7 +20,6 @@ #include #include -#include #include @@ -164,7 +163,7 @@ public: * Use only after all calls to joinBlock was done. * left_sample_block is passed without account of 'use_nulls' setting (columns will be converted to Nullable inside). */ - BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const override; + std::shared_ptr getNonJoinedBlocks(const Block & result_sample_block, UInt64 max_block_size) const override; /// Number of keys in all built JOIN maps. size_t getTotalRowCount() const final; diff --git a/src/Interpreters/IJoin.h b/src/Interpreters/IJoin.h index 8fa85de4951..2215402e1d4 100644 --- a/src/Interpreters/IJoin.h +++ b/src/Interpreters/IJoin.h @@ -5,7 +5,6 @@ #include #include -#include namespace DB { @@ -15,6 +14,7 @@ struct ExtraBlock; using ExtraBlockPtr = std::shared_ptr; class TableJoin; +class NotJoinedBlocks; class IJoin { @@ -43,7 +43,7 @@ public: /// Different query plan is used for such joins. virtual bool isFilled() const { return false; } - virtual BlockInputStreamPtr createStreamWithNonJoinedRows(const Block &, UInt64) const { return {}; } + virtual std::shared_ptr getNonJoinedBlocks(const Block &, UInt64) const = 0; }; using JoinPtr = std::shared_ptr; diff --git a/src/Interpreters/JoinSwitcher.h b/src/Interpreters/JoinSwitcher.h index a89ac6d5d98..e750bc5eed0 100644 --- a/src/Interpreters/JoinSwitcher.h +++ b/src/Interpreters/JoinSwitcher.h @@ -56,9 +56,9 @@ public: return join->alwaysReturnsEmptySet(); } - BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & block, UInt64 max_block_size) const override + std::shared_ptr getNonJoinedBlocks(const Block & block, UInt64 max_block_size) const override { - return join->createStreamWithNonJoinedRows(block, max_block_size); + return join->getNonJoinedBlocks(block, max_block_size); } private: @@ -74,38 +74,4 @@ private: void switchJoin(); }; - -/// Creates NonJoinedBlockInputStream on the first read. Allows to swap join algo before it. -class LazyNonJoinedBlockInputStream : public IBlockInputStream -{ -public: - LazyNonJoinedBlockInputStream(const IJoin & join_, const Block & block, UInt64 max_block_size_) - : join(join_) - , result_sample_block(block) - , max_block_size(max_block_size_) - {} - - String getName() const override { return "LazyNonMergeJoined"; } - Block getHeader() const override { return result_sample_block; } - -protected: - Block readImpl() override - { - if (!stream) - { - stream = join.createStreamWithNonJoinedRows(result_sample_block, max_block_size); - if (!stream) - return {}; - } - - return stream->read(); - } - -private: - BlockInputStreamPtr stream; - const IJoin & join; - Block result_sample_block; - UInt64 max_block_size; -}; - } diff --git a/src/Interpreters/MergeJoin.cpp b/src/Interpreters/MergeJoin.cpp index 611f1742fa4..0150bbe1d93 100644 --- a/src/Interpreters/MergeJoin.cpp +++ b/src/Interpreters/MergeJoin.cpp @@ -1028,7 +1028,7 @@ void MergeJoin::initRightTableWriter() } /// Stream from not joined earlier rows of the right table. -class NotJoinedMerge final : public NotJoinedInputStream::RightColumnsFiller +class NotJoinedMerge final : public NotJoinedBlocks::RightColumnsFiller { public: NotJoinedMerge(const MergeJoin & parent_, UInt64 max_block_size_) @@ -1089,15 +1089,15 @@ private: }; -BlockInputStreamPtr MergeJoin::createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const +std::shared_ptr MergeJoin::getNonJoinedBlocks(const Block & result_sample_block, UInt64 max_block_size) const { if (table_join->strictness() == ASTTableJoin::Strictness::All && (is_right || is_full)) { size_t left_columns_count = result_sample_block.columns() - right_columns_to_add.columns(); auto non_joined = std::make_unique(*this, max_block_size); - return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); + return std::make_shared(std::move(non_joined), result_sample_block, left_columns_count, table_join->leftToRightKeyRemap()); } - return {}; + return nullptr; } bool MergeJoin::needConditionJoinColumn() const diff --git a/src/Interpreters/MergeJoin.h b/src/Interpreters/MergeJoin.h index 89371d8b13b..9e765041846 100644 --- a/src/Interpreters/MergeJoin.h +++ b/src/Interpreters/MergeJoin.h @@ -35,7 +35,7 @@ public: /// Has to be called only after setTotals()/mergeRightBlocks() bool alwaysReturnsEmptySet() const override { return (is_right || is_inner) && min_max_right_blocks.empty(); } - BlockInputStreamPtr createStreamWithNonJoinedRows(const Block & result_sample_block, UInt64 max_block_size) const override; + std::shared_ptr getNonJoinedBlocks(const Block & result_sample_block, UInt64 max_block_size) const override; private: friend class NotJoinedMerge; diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index c640fea3a36..349ba56e74a 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -492,7 +492,7 @@ void splitAdditionalColumns(const Names & key_names, const Block & sample_block, } -NotJoinedInputStream::NotJoinedInputStream(std::unique_ptr filler_, +NotJoinedBlocks::NotJoinedBlocks(std::unique_ptr filler_, const Block & result_sample_block_, size_t left_columns_count, const LeftToRightKeyRemap & left_to_right_key_remap) @@ -537,7 +537,7 @@ NotJoinedInputStream::NotJoinedInputStream(std::unique_ptr f ErrorCodes::LOGICAL_ERROR); } -void NotJoinedInputStream::setRightIndex(size_t right_pos, size_t result_position) +void NotJoinedBlocks::setRightIndex(size_t right_pos, size_t result_position) { if (!column_indices_right.contains(right_pos)) { @@ -548,7 +548,7 @@ void NotJoinedInputStream::setRightIndex(size_t right_pos, size_t result_positio same_result_keys[result_position] = column_indices_right[right_pos]; } -void NotJoinedInputStream::extractColumnChanges(size_t right_pos, size_t result_pos) +void NotJoinedBlocks::extractColumnChanges(size_t right_pos, size_t result_pos) { auto src_props = getLowcardAndNullability(saved_block_sample.getByPosition(right_pos).column); auto dst_props = getLowcardAndNullability(result_sample_block.getByPosition(result_pos).column); @@ -560,7 +560,7 @@ void NotJoinedInputStream::extractColumnChanges(size_t right_pos, size_t result_ right_lowcard_changes.push_back({result_pos, dst_props.is_lowcard}); } -void NotJoinedInputStream::correctLowcardAndNullability(Block & block) +void NotJoinedBlocks::correctLowcardAndNullability(Block & block) { for (auto & [pos, added] : right_nullability_changes) { @@ -588,7 +588,7 @@ void NotJoinedInputStream::correctLowcardAndNullability(Block & block) } } -void NotJoinedInputStream::addLeftColumns(Block & block, size_t rows_added) const +void NotJoinedBlocks::addLeftColumns(Block & block, size_t rows_added) const { for (size_t pos : column_indices_left) { @@ -600,7 +600,7 @@ void NotJoinedInputStream::addLeftColumns(Block & block, size_t rows_added) cons } } -void NotJoinedInputStream::addRightColumns(Block & block, MutableColumns & columns_right) const +void NotJoinedBlocks::addRightColumns(Block & block, MutableColumns & columns_right) const { for (const auto & pr : column_indices_right) { @@ -610,7 +610,7 @@ void NotJoinedInputStream::addRightColumns(Block & block, MutableColumns & colum } } -void NotJoinedInputStream::copySameKeys(Block & block) const +void NotJoinedBlocks::copySameKeys(Block & block) const { for (const auto & pr : same_result_keys) { @@ -620,7 +620,7 @@ void NotJoinedInputStream::copySameKeys(Block & block) const } } -Block NotJoinedInputStream::readImpl() +Block NotJoinedBlocks::read() { Block right_block = filler->getEmptyBlock(); @@ -635,7 +635,7 @@ Block NotJoinedInputStream::readImpl() correctLowcardAndNullability(right_block); #ifndef NDEBUG - assertBlocksHaveEqualStructure(right_block, result_sample_block, getName()); + assertBlocksHaveEqualStructure(right_block, result_sample_block, "NotJoinedBlocks"); #endif return right_block; } diff --git a/src/Interpreters/join_common.h b/src/Interpreters/join_common.h index 32fa4a4ee9e..ec2e1d3bd50 100644 --- a/src/Interpreters/join_common.h +++ b/src/Interpreters/join_common.h @@ -5,7 +5,6 @@ #include #include #include -#include namespace DB { @@ -65,7 +64,7 @@ void changeLowCardinalityInplace(ColumnWithTypeAndName & column); } /// Creates result from right table data in RIGHT and FULL JOIN when keys are not present in left table. -class NotJoinedInputStream : public IBlockInputStream +class NotJoinedBlocks final { public: using LeftToRightKeyRemap = std::unordered_map; @@ -82,16 +81,12 @@ public: virtual ~RightColumnsFiller() = default; }; - NotJoinedInputStream(std::unique_ptr filler_, + NotJoinedBlocks(std::unique_ptr filler_, const Block & result_sample_block_, size_t left_columns_count, const LeftToRightKeyRemap & left_to_right_key_remap); - String getName() const override { return "NonJoined"; } - Block getHeader() const override { return result_sample_block; } - -protected: - Block readImpl() override final; + Block read(); private: void extractColumnChanges(size_t right_pos, size_t result_pos); diff --git a/src/Processors/Transforms/JoiningTransform.cpp b/src/Processors/Transforms/JoiningTransform.cpp index c1329d02fed..95a12e2291d 100644 --- a/src/Processors/Transforms/JoiningTransform.cpp +++ b/src/Processors/Transforms/JoiningTransform.cpp @@ -113,7 +113,7 @@ void JoiningTransform::work() } else { - if (!non_joined_stream) + if (!non_joined_blocks) { if (!finish_counter || !finish_counter->isLast()) { @@ -121,15 +121,15 @@ void JoiningTransform::work() return; } - non_joined_stream = join->createStreamWithNonJoinedRows(outputs.front().getHeader(), max_block_size); - if (!non_joined_stream) + non_joined_blocks = join->getNonJoinedBlocks(outputs.front().getHeader(), max_block_size); + if (!non_joined_blocks) { process_non_joined = false; return; } } - auto block = non_joined_stream->read(); + Block block = non_joined_blocks->read(); if (!block) { process_non_joined = false; diff --git a/src/Processors/Transforms/JoiningTransform.h b/src/Processors/Transforms/JoiningTransform.h index 98038946f3b..96c4032dabc 100644 --- a/src/Processors/Transforms/JoiningTransform.h +++ b/src/Processors/Transforms/JoiningTransform.h @@ -8,8 +8,7 @@ namespace DB class IJoin; using JoinPtr = std::shared_ptr; -class IBlockInputStream; -using BlockInputStreamPtr = std::shared_ptr; +class NotJoinedBlocks; /// Join rows to chunk form left table. /// This transform usually has two input ports and one output. @@ -76,7 +75,7 @@ private: ExtraBlockPtr not_processed; FinishCounterPtr finish_counter; - BlockInputStreamPtr non_joined_stream; + std::shared_ptr non_joined_blocks; size_t max_block_size; Block readExecute(Chunk & chunk); From c36569e17c900cf51a7145303abc6da08098849e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 17 Aug 2021 16:33:30 +0300 Subject: [PATCH 133/220] Try fix integration tests. --- .../configs/dictionaries/postgres_dict.xml | 8 +-- .../test_dictionaries_postgresql/test.py | 14 ++--- .../postgres_odbc_hashed_dictionary.xml | 2 +- .../sqlite3_odbc_cached_dictionary.xml | 2 +- .../sqlite3_odbc_hashed_dictionary.xml | 2 +- .../integration/test_odbc_interaction/test.py | 60 +++++++++---------- 6 files changed, 44 insertions(+), 44 deletions(-) diff --git a/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml index 4ee07d0972a..734da0cff70 100644 --- a/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml +++ b/tests/integration/test_dictionaries_postgresql/configs/dictionaries/postgres_dict.xml @@ -19,10 +19,10 @@ id - UInt32 + - id + key UInt32 @@ -65,10 +65,10 @@ id - UInt32 + - id + key UInt32 diff --git a/tests/integration/test_dictionaries_postgresql/test.py b/tests/integration/test_dictionaries_postgresql/test.py index 6eb4a04ed2c..58a503bd571 100644 --- a/tests/integration/test_dictionaries_postgresql/test.py +++ b/tests/integration/test_dictionaries_postgresql/test.py @@ -13,11 +13,11 @@ node1 = cluster.add_instance('node1', postgres_dict_table_template = """ CREATE TABLE IF NOT EXISTS {} ( - id Integer NOT NULL, value Integer NOT NULL, PRIMARY KEY (id)) + id Integer NOT NULL, key Integer NOT NULL, value Integer NOT NULL, PRIMARY KEY (id)) """ click_dict_table_template = """ CREATE TABLE IF NOT EXISTS `test`.`dict_table_{}` ( - `id` UInt64, `value` UInt32 + `key` UInt32, `value` UInt32 ) ENGINE = Dictionary({}) """ @@ -43,7 +43,7 @@ def create_and_fill_postgres_table(cursor, table_name, port, host): create_postgres_table(cursor, table_name) # Fill postgres table using clickhouse postgres table function and check table_func = '''postgresql('{}:{}', 'clickhouse', '{}', 'postgres', 'mysecretpassword')'''.format(host, port, table_name) - node1.query('''INSERT INTO TABLE FUNCTION {} SELECT number, number from numbers(10000) + node1.query('''INSERT INTO TABLE FUNCTION {} SELECT number, number, number from numbers(10000) '''.format(table_func, table_name)) result = node1.query("SELECT count() FROM {}".format(table_func)) assert result.rstrip() == '10000' @@ -82,7 +82,7 @@ def test_load_dictionaries(started_cluster): node1.query("SYSTEM RELOAD DICTIONARY {}".format(dict_name)) assert node1.query("SELECT count() FROM `test`.`dict_table_{}`".format(table_name)).rstrip() == '10000' - assert node1.query("SELECT dictGetUInt32('{}', 'id', toUInt64(0))".format(dict_name)) == '0\n' + assert node1.query("SELECT dictGetUInt32('{}', 'key', toUInt64(0))".format(dict_name)) == '0\n' assert node1.query("SELECT dictGetUInt32('{}', 'value', toUInt64(9999))".format(dict_name)) == '9999\n' cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) @@ -252,11 +252,11 @@ def test_dictionary_with_replicas(started_cluster): create_postgres_table(cursor1, 'test1') create_postgres_table(cursor2, 'test1') - cursor1.execute('INSERT INTO test1 select i, i from generate_series(0, 99) as t(i);'); - cursor2.execute('INSERT INTO test1 select i, i from generate_series(100, 199) as t(i);'); + cursor1.execute('INSERT INTO test1 select i, i, i from generate_series(0, 99) as t(i);') + cursor2.execute('INSERT INTO test1 select i, i, i from generate_series(100, 199) as t(i);') create_dict('test1', 1) - result = node1.query("SELECT * FROM `test`.`dict_table_test1` ORDER BY id") + result = node1.query("SELECT * FROM `test`.`dict_table_test1` ORDER BY key") # priority 0 - non running port assert node1.contains_in_log('PostgreSQLConnectionPool: Connection error*') diff --git a/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_hashed_dictionary.xml b/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_hashed_dictionary.xml index 6aad3ad9917..a65360b0e26 100644 --- a/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_hashed_dictionary.xml +++ b/tests/integration/test_odbc_interaction/configs/dictionaries/postgres_odbc_hashed_dictionary.xml @@ -18,7 +18,7 @@ - column1 + id diff --git a/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_cached_dictionary.xml b/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_cached_dictionary.xml index 45f3966ee8a..3a505b79304 100644 --- a/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_cached_dictionary.xml +++ b/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_cached_dictionary.xml @@ -20,7 +20,7 @@ - X + id diff --git a/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml b/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml index 18a14b896bd..5b53818cf13 100644 --- a/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml +++ b/tests/integration/test_odbc_interaction/configs/dictionaries/sqlite3_odbc_hashed_dictionary.xml @@ -20,7 +20,7 @@ - X + id diff --git a/tests/integration/test_odbc_interaction/test.py b/tests/integration/test_odbc_interaction/test.py index 39a283448f5..4d2f70ad08c 100644 --- a/tests/integration/test_odbc_interaction/test.py +++ b/tests/integration/test_odbc_interaction/test.py @@ -99,19 +99,19 @@ def started_cluster(): logging.debug(f"sqlite data received: {sqlite_db}") node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t1(x INTEGER PRIMARY KEY ASC, y, z);"], + ["sqlite3", sqlite_db, "CREATE TABLE t1(id INTEGER PRIMARY KEY ASC, x INTEGER, y, z);"], privileged=True, user='root') node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t2(X INTEGER PRIMARY KEY ASC, Y, Z);"], + ["sqlite3", sqlite_db, "CREATE TABLE t2(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], privileged=True, user='root') node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t3(X INTEGER PRIMARY KEY ASC, Y, Z);"], + ["sqlite3", sqlite_db, "CREATE TABLE t3(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], privileged=True, user='root') node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE t4(X INTEGER PRIMARY KEY ASC, Y, Z);"], + ["sqlite3", sqlite_db, "CREATE TABLE t4(id INTEGER PRIMARY KEY ASC, X INTEGER, Y, Z);"], privileged=True, user='root') node1.exec_in_container( - ["sqlite3", sqlite_db, "CREATE TABLE tf1(x INTEGER PRIMARY KEY ASC, y, z);"], + ["sqlite3", sqlite_db, "CREATE TABLE tf1(id INTEGER PRIMARY KEY ASC, x INTEGER, y, z);"], privileged=True, user='root') logging.debug("sqlite tables created") mysql_conn = get_mysql_conn() @@ -128,7 +128,7 @@ def started_cluster(): cursor = postgres_conn.cursor() cursor.execute( - "create table if not exists clickhouse.test_table (column1 int primary key, column2 varchar(40) not null)") + "create table if not exists clickhouse.test_table (id int primary key, column1 int not null, column2 varchar(40) not null)") yield cluster @@ -210,9 +210,9 @@ def test_sqlite_simple_select_function_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t1 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t1 values(1, 1, 2, 3);"], privileged=True, user='root') - assert node1.query("select * from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\t2\t3\n" + assert node1.query("select * from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "1\t1\t2\t3\n" assert node1.query("select y from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "2\n" assert node1.query("select z from odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 't1')) == "3\n" @@ -228,10 +228,10 @@ def test_sqlite_table_function(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO tf1 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO tf1 values(1, 1, 2, 3);"], privileged=True, user='root') node1.query("create table odbc_tf as odbc('DSN={}', '{}')".format(sqlite_setup["DSN"], 'tf1')) - assert node1.query("select * from odbc_tf") == "1\t2\t3\n" + assert node1.query("select * from odbc_tf") == "1\t1\t2\t3\n" assert node1.query("select y from odbc_tf") == "2\n" assert node1.query("select z from odbc_tf") == "3\n" @@ -246,7 +246,7 @@ def test_sqlite_simple_select_storage_works(started_cluster): sqlite_setup = node1.odbc_drivers["SQLite3"] sqlite_db = sqlite_setup["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t4 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t4 values(1, 1, 2, 3);"], privileged=True, user='root') node1.query("create table SqliteODBC (x Int32, y String, z String) engine = ODBC('DSN={}', '', 't4')".format( sqlite_setup["DSN"])) @@ -264,7 +264,7 @@ def test_sqlite_odbc_hashed_dictionary(started_cluster): skip_test_msan(node1) sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(1, 1, 2, 3);"], privileged=True, user='root') node1.query("SYSTEM RELOAD DICTIONARY sqlite3_odbc_hashed") @@ -282,7 +282,7 @@ def test_sqlite_odbc_hashed_dictionary(started_cluster): logging.debug("Waiting dictionary to update for the second time") time.sleep(0.1) - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(200, 2, 7);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t2 values(200, 200, 2, 7);"], privileged=True, user='root') # No reload because of invalidate query @@ -299,7 +299,7 @@ def test_sqlite_odbc_hashed_dictionary(started_cluster): assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "3") assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(200))", "1") # still default - node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t2 values(1, 2, 5);"], + node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t2 values(1, 1, 2, 5);"], privileged=True, user='root') assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_hashed', 'Z', toUInt64(1))", "5") @@ -310,7 +310,7 @@ def test_sqlite_odbc_cached_dictionary(started_cluster): skip_test_msan(node1) sqlite_db = node1.odbc_drivers["SQLite3"]["Database"] - node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t3 values(1, 2, 3);"], + node1.exec_in_container(["sqlite3", sqlite_db, "INSERT INTO t3 values(1, 1, 2, 3);"], privileged=True, user='root') assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))") == "3\n" @@ -319,12 +319,12 @@ def test_sqlite_odbc_cached_dictionary(started_cluster): node1.exec_in_container(["chmod", "a+rw", "/tmp"], privileged=True, user='root') node1.exec_in_container(["chmod", "a+rw", sqlite_db], privileged=True, user='root') - node1.query("insert into table function odbc('DSN={};ReadOnly=0', '', 't3') values (200, 2, 7)".format( + node1.query("insert into table function odbc('DSN={};ReadOnly=0', '', 't3') values (200, 200, 2, 7)".format( node1.odbc_drivers["SQLite3"]["DSN"])) assert node1.query("select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(200))") == "7\n" # new value - node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t3 values(1, 2, 12);"], + node1.exec_in_container(["sqlite3", sqlite_db, "REPLACE INTO t3 values(1, 1, 2, 12);"], privileged=True, user='root') assert_eq_with_retry(node1, "select dictGetUInt8('sqlite3_odbc_cached', 'Z', toUInt64(1))", "12") @@ -336,7 +336,7 @@ def test_postgres_odbc_hashed_dictionary_with_schema(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() cursor.execute("truncate table clickhouse.test_table") - cursor.execute("insert into clickhouse.test_table values(1, 'hello'),(2, 'world')") + cursor.execute("insert into clickhouse.test_table values(1, 1, 'hello'),(2, 2, 'world')") node1.query("SYSTEM RELOAD DICTIONARY postgres_odbc_hashed") assert_eq_with_retry(node1, "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(1))", "hello") assert_eq_with_retry(node1, "select dictGetString('postgres_odbc_hashed', 'column2', toUInt64(2))", "world") @@ -348,7 +348,7 @@ def test_postgres_odbc_hashed_dictionary_no_tty_pipe_overflow(started_cluster): conn = get_postgres_conn(started_cluster) cursor = conn.cursor() cursor.execute("truncate table clickhouse.test_table") - cursor.execute("insert into clickhouse.test_table values(3, 'xxx')") + cursor.execute("insert into clickhouse.test_table values(3, 3, 'xxx')") for i in range(100): try: node1.query("system reload dictionary postgres_odbc_hashed", timeout=15) @@ -369,13 +369,13 @@ def test_postgres_insert(started_cluster): # reconstruction of connection string. node1.query( - "create table pg_insert (column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')") - node1.query("insert into pg_insert values (1, 'hello'), (2, 'world')") - assert node1.query("select * from pg_insert") == '1\thello\n2\tworld\n' - node1.query("insert into table function odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table') format CSV 3,test") + "create table pg_insert (id UInt64, column1 UInt8, column2 String) engine=ODBC('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')") + node1.query("insert into pg_insert values (1, 1, 'hello'), (2, 2, 'world')") + assert node1.query("select * from pg_insert") == '1\t1\thello\n2\t2\tworld\n' + node1.query("insert into table function odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table') format CSV 3,3,test") node1.query( "insert into table function odbc('DSN=postgresql_odbc;Servername=postgre-sql.local', 'clickhouse', 'test_table')" \ - " select number, 's' || toString(number) from numbers (4, 7)") + " select number, number, 's' || toString(number) from numbers (4, 7)") assert node1.query("select sum(column1), count(column1) from pg_insert") == "55\t10\n" assert node1.query( "select sum(n), count(n) from (select (*,).1 as n from (select * from odbc('DSN=postgresql_odbc', 'clickhouse', 'test_table')))") == "55\t10\n" @@ -426,19 +426,19 @@ def test_odbc_postgres_date_data_type(started_cluster): conn = get_postgres_conn(started_cluster); cursor = conn.cursor() - cursor.execute("CREATE TABLE IF NOT EXISTS clickhouse.test_date (column1 integer, column2 date)") + cursor.execute("CREATE TABLE IF NOT EXISTS clickhouse.test_date (id integer, column1 integer, column2 date)") - cursor.execute("INSERT INTO clickhouse.test_date VALUES (1, '2020-12-01')") - cursor.execute("INSERT INTO clickhouse.test_date VALUES (2, '2020-12-02')") - cursor.execute("INSERT INTO clickhouse.test_date VALUES (3, '2020-12-03')") + cursor.execute("INSERT INTO clickhouse.test_date VALUES (1, 1, '2020-12-01')") + cursor.execute("INSERT INTO clickhouse.test_date VALUES (2, 2, '2020-12-02')") + cursor.execute("INSERT INTO clickhouse.test_date VALUES (3, 3, '2020-12-03')") conn.commit() node1.query( ''' - CREATE TABLE test_date (column1 UInt64, column2 Date) + CREATE TABLE test_date (id UInt64, column1 UInt64, column2 Date) ENGINE=ODBC('DSN=postgresql_odbc; Servername=postgre-sql.local', 'clickhouse', 'test_date')''') - expected = '1\t2020-12-01\n2\t2020-12-02\n3\t2020-12-03\n' + expected = '1\t1\t2020-12-01\n2\t2\t2020-12-02\n3\t3\t2020-12-03\n' result = node1.query('SELECT * FROM test_date'); assert(result == expected) cursor.execute("DROP TABLE IF EXISTS clickhouse.test_date") From 3fc0e577e3d87cddee7ed29e6188e2d72b5fa672 Mon Sep 17 00:00:00 2001 From: Filatenkov Artur <58165623+FArthur-cmd@users.noreply.github.com> Date: Tue, 17 Aug 2021 17:22:38 +0300 Subject: [PATCH 134/220] remove trailing whitespaces --- src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp index 75750211907..daf07a208bd 100644 --- a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp @@ -64,7 +64,7 @@ Pipe getSourceFromFromASTInsertQuery( const auto in_file = in_file_node.value.safeGet(); input_buffer = wrapReadBufferWithCompressionMethod(std::make_unique(in_file), chooseCompressionMethod(in_file, "")); - } else + } else { ConcatReadBuffer::ReadBuffers buffers; if (ast_insert_query->data) From d346d19942083310306e51db6fac923516fce2d1 Mon Sep 17 00:00:00 2001 From: Kevin Michel Date: Tue, 20 Jul 2021 17:07:54 +0200 Subject: [PATCH 135/220] Extract AccessEntity parsing from DiskAccessStorage --- src/Access/AccessEntityIO.cpp | 175 +++++++++++++++++++++++++++++++ src/Access/AccessEntityIO.h | 12 +++ src/Access/DiskAccessStorage.cpp | 143 +------------------------ src/Access/ya.make | 1 + 4 files changed, 191 insertions(+), 140 deletions(-) create mode 100644 src/Access/AccessEntityIO.cpp create mode 100644 src/Access/AccessEntityIO.h diff --git a/src/Access/AccessEntityIO.cpp b/src/Access/AccessEntityIO.cpp new file mode 100644 index 00000000000..2160f3e9db8 --- /dev/null +++ b/src/Access/AccessEntityIO.cpp @@ -0,0 +1,175 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_ACCESS_ENTITY_DEFINITION; +} + +using EntityType = IAccessStorage::EntityType; +using EntityTypeInfo = IAccessStorage::EntityTypeInfo; + +namespace +{ + /// Special parser for the 'ATTACH access entity' queries. + class ParserAttachAccessEntity : public IParserBase + { + protected: + const char * getName() const override { return "ATTACH access entity query"; } + + bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override + { + ParserCreateUserQuery create_user_p; + ParserCreateRoleQuery create_role_p; + ParserCreateRowPolicyQuery create_policy_p; + ParserCreateQuotaQuery create_quota_p; + ParserCreateSettingsProfileQuery create_profile_p; + ParserGrantQuery grant_p; + + create_user_p.useAttachMode(); + create_role_p.useAttachMode(); + create_policy_p.useAttachMode(); + create_quota_p.useAttachMode(); + create_profile_p.useAttachMode(); + grant_p.useAttachMode(); + + return create_user_p.parse(pos, node, expected) || create_role_p.parse(pos, node, expected) + || create_policy_p.parse(pos, node, expected) || create_quota_p.parse(pos, node, expected) + || create_profile_p.parse(pos, node, expected) || grant_p.parse(pos, node, expected); + } + }; + +} + + +String serializeAccessEntity(const IAccessEntity & entity) +{ + /// Build list of ATTACH queries. + ASTs queries; + queries.push_back(InterpreterShowCreateAccessEntityQuery::getAttachQuery(entity)); + if ((entity.getType() == EntityType::USER) || (entity.getType() == EntityType::ROLE)) + boost::range::push_back(queries, InterpreterShowGrantsQuery::getAttachGrantQueries(entity)); + + /// Serialize the list of ATTACH queries to a string. + WriteBufferFromOwnString buf; + for (const ASTPtr & query : queries) + { + formatAST(*query, buf, false, true); + buf.write(";\n", 2); + } + return buf.str(); +} + +AccessEntityPtr deserializeAccessEntity(const String & definition, const String & path) +{ + ASTs queries; + ParserAttachAccessEntity parser; + const char * begin = definition.data(); /// begin of current query + const char * pos = begin; /// parser moves pos from begin to the end of current query + const char * end = begin + definition.size(); + while (pos < end) + { + queries.emplace_back(parseQueryAndMovePosition(parser, pos, end, "", true, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH)); + while (isWhitespaceASCII(*pos) || *pos == ';') + ++pos; + } + + /// Interpret the AST to build an access entity. + std::shared_ptr user; + std::shared_ptr role; + std::shared_ptr policy; + std::shared_ptr quota; + std::shared_ptr profile; + AccessEntityPtr res; + + for (const auto & query : queries) + { + if (auto * create_user_query = query->as()) + { + if (res) + throw Exception("Two access entities attached in " + path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); + res = user = std::make_unique(); + InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query); + } + else if (auto * create_role_query = query->as()) + { + if (res) + throw Exception("Two access entities attached in " + path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); + res = role = std::make_unique(); + InterpreterCreateRoleQuery::updateRoleFromQuery(*role, *create_role_query); + } + else if (auto * create_policy_query = query->as()) + { + if (res) + throw Exception("Two access entities attached in " + path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); + res = policy = std::make_unique(); + InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(*policy, *create_policy_query); + } + else if (auto * create_quota_query = query->as()) + { + if (res) + throw Exception("Two access entities attached in " + path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); + res = quota = std::make_unique(); + InterpreterCreateQuotaQuery::updateQuotaFromQuery(*quota, *create_quota_query); + } + else if (auto * create_profile_query = query->as()) + { + if (res) + throw Exception("Two access entities attached in " + path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); + res = profile = std::make_unique(); + InterpreterCreateSettingsProfileQuery::updateSettingsProfileFromQuery(*profile, *create_profile_query); + } + else if (auto * grant_query = query->as()) + { + if (!user && !role) + throw Exception( + "A user or role should be attached before grant in " + path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); + if (user) + InterpreterGrantQuery::updateUserFromQuery(*user, *grant_query); + else + InterpreterGrantQuery::updateRoleFromQuery(*role, *grant_query); + } + else + throw Exception("No interpreter found for query " + query->getID(), ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); + } + + if (!res) + throw Exception("No access entities attached in " + path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); + + return res; +} + +} diff --git a/src/Access/AccessEntityIO.h b/src/Access/AccessEntityIO.h new file mode 100644 index 00000000000..94dc027430e --- /dev/null +++ b/src/Access/AccessEntityIO.h @@ -0,0 +1,12 @@ +#pragma once + +#include + +namespace DB +{ + +String serializeAccessEntity(const IAccessEntity & entity); + +AccessEntityPtr deserializeAccessEntity(const String & definition, const String & path); + +} diff --git a/src/Access/DiskAccessStorage.cpp b/src/Access/DiskAccessStorage.cpp index 2c074e3d0b0..5fb6f48e1ee 100644 --- a/src/Access/DiskAccessStorage.cpp +++ b/src/Access/DiskAccessStorage.cpp @@ -4,41 +4,20 @@ #include #include #include +#include #include #include #include #include -#include #include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include #include -#include #include -#include -#include -#include -#include -#include -#include #include #include -#include #include #include #include #include -#include -#include #include #include @@ -49,7 +28,6 @@ namespace ErrorCodes { extern const int DIRECTORY_DOESNT_EXIST; extern const int FILE_DOESNT_EXIST; - extern const int INCORRECT_ACCESS_ENTITY_DEFINITION; } @@ -58,34 +36,6 @@ namespace using EntityType = IAccessStorage::EntityType; using EntityTypeInfo = IAccessStorage::EntityTypeInfo; - /// Special parser for the 'ATTACH access entity' queries. - class ParserAttachAccessEntity : public IParserBase - { - protected: - const char * getName() const override { return "ATTACH access entity query"; } - - bool parseImpl(Pos & pos, ASTPtr & node, Expected & expected) override - { - ParserCreateUserQuery create_user_p; - ParserCreateRoleQuery create_role_p; - ParserCreateRowPolicyQuery create_policy_p; - ParserCreateQuotaQuery create_quota_p; - ParserCreateSettingsProfileQuery create_profile_p; - ParserGrantQuery grant_p; - - create_user_p.useAttachMode(); - create_role_p.useAttachMode(); - create_policy_p.useAttachMode(); - create_quota_p.useAttachMode(); - create_profile_p.useAttachMode(); - grant_p.useAttachMode(); - - return create_user_p.parse(pos, node, expected) || create_role_p.parse(pos, node, expected) - || create_policy_p.parse(pos, node, expected) || create_quota_p.parse(pos, node, expected) - || create_profile_p.parse(pos, node, expected) || grant_p.parse(pos, node, expected); - } - }; - /// Reads a file containing ATTACH queries and then parses it to build an access entity. AccessEntityPtr readEntityFile(const String & file_path) @@ -96,80 +46,7 @@ namespace readStringUntilEOF(file_contents, in); /// Parse the file contents. - ASTs queries; - ParserAttachAccessEntity parser; - const char * begin = file_contents.data(); /// begin of current query - const char * pos = begin; /// parser moves pos from begin to the end of current query - const char * end = begin + file_contents.size(); - while (pos < end) - { - queries.emplace_back(parseQueryAndMovePosition(parser, pos, end, "", true, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH)); - while (isWhitespaceASCII(*pos) || *pos == ';') - ++pos; - } - - /// Interpret the AST to build an access entity. - std::shared_ptr user; - std::shared_ptr role; - std::shared_ptr policy; - std::shared_ptr quota; - std::shared_ptr profile; - AccessEntityPtr res; - - for (const auto & query : queries) - { - if (auto * create_user_query = query->as()) - { - if (res) - throw Exception("Two access entities in one file " + file_path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); - res = user = std::make_unique(); - InterpreterCreateUserQuery::updateUserFromQuery(*user, *create_user_query); - } - else if (auto * create_role_query = query->as()) - { - if (res) - throw Exception("Two access entities in one file " + file_path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); - res = role = std::make_unique(); - InterpreterCreateRoleQuery::updateRoleFromQuery(*role, *create_role_query); - } - else if (auto * create_policy_query = query->as()) - { - if (res) - throw Exception("Two access entities in one file " + file_path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); - res = policy = std::make_unique(); - InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(*policy, *create_policy_query); - } - else if (auto * create_quota_query = query->as()) - { - if (res) - throw Exception("Two access entities are attached in the same file " + file_path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); - res = quota = std::make_unique(); - InterpreterCreateQuotaQuery::updateQuotaFromQuery(*quota, *create_quota_query); - } - else if (auto * create_profile_query = query->as()) - { - if (res) - throw Exception("Two access entities are attached in the same file " + file_path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); - res = profile = std::make_unique(); - InterpreterCreateSettingsProfileQuery::updateSettingsProfileFromQuery(*profile, *create_profile_query); - } - else if (auto * grant_query = query->as()) - { - if (!user && !role) - throw Exception("A user or role should be attached before grant in file " + file_path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); - if (user) - InterpreterGrantQuery::updateUserFromQuery(*user, *grant_query); - else - InterpreterGrantQuery::updateRoleFromQuery(*role, *grant_query); - } - else - throw Exception("No interpreter found for query " + query->getID(), ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); - } - - if (!res) - throw Exception("No access entities attached in file " + file_path, ErrorCodes::INCORRECT_ACCESS_ENTITY_DEFINITION); - - return res; + return deserializeAccessEntity(file_contents, file_path); } @@ -186,24 +63,10 @@ namespace } } - /// Writes ATTACH queries for building a specified access entity to a file. void writeEntityFile(const String & file_path, const IAccessEntity & entity) { - /// Build list of ATTACH queries. - ASTs queries; - queries.push_back(InterpreterShowCreateAccessEntityQuery::getAttachQuery(entity)); - if ((entity.getType() == EntityType::USER) || (entity.getType() == EntityType::ROLE)) - boost::range::push_back(queries, InterpreterShowGrantsQuery::getAttachGrantQueries(entity)); - - /// Serialize the list of ATTACH queries to a string. - WriteBufferFromOwnString buf; - for (const ASTPtr & query : queries) - { - formatAST(*query, buf, false, true); - buf.write(";\n", 2); - } - String file_contents = buf.str(); + String file_contents = serializeAccessEntity(entity); /// First we save *.tmp file and then we rename if everything's ok. auto tmp_file_path = std::filesystem::path{file_path}.replace_extension(".tmp"); diff --git a/src/Access/ya.make b/src/Access/ya.make index 5f2f410cabd..3ac4c944f2a 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -10,6 +10,7 @@ PEERDIR( SRCS( AccessControlManager.cpp + AccessEntityIO.cpp AccessRights.cpp AccessRightsElement.cpp AllowedClientHosts.cpp From 90294e6dd8fb075b2ade445af301aeef7c1f664e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:22:41 +0200 Subject: [PATCH 136/220] 01889_sqlite_read_write: Made parallelizable and cleanup properly --- .../0_stateless/01889_sqlite_read_write.sh | 82 ++++++++++--------- tests/queries/skip_list.json | 1 - 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/tests/queries/0_stateless/01889_sqlite_read_write.sh b/tests/queries/0_stateless/01889_sqlite_read_write.sh index 73b106e9eb4..3e7e15d2830 100755 --- a/tests/queries/0_stateless/01889_sqlite_read_write.sh +++ b/tests/queries/0_stateless/01889_sqlite_read_write.sh @@ -7,60 +7,68 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # See 01658_read_file_to_string_column.sh user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') -mkdir -p ${user_files_path}/ -chmod 777 ${user_files_path} -DB_PATH=${user_files_path}/db1 +mkdir -p "${user_files_path}/" +chmod 777 "${user_files_path}" +export CURR_DATABASE="test_01889_sqllite_${CLICKHOUSE_DATABASE}" -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table1' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table2' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table3' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table4' -sqlite3 ${DB_PATH} 'DROP TABLE IF EXISTS table5' +DB_PATH=${user_files_path}/${CURR_DATABASE}_db1 +DB_PATH2=$CUR_DIR/${CURR_DATABASE}_db2 -sqlite3 ${DB_PATH} 'CREATE TABLE table1 (col1 text, col2 smallint);' -sqlite3 ${DB_PATH} 'CREATE TABLE table2 (col1 int, col2 text);' +function cleanup() +{ + ${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS ${CURR_DATABASE}" + rm -r "${DB_PATH}" "${DB_PATH2}" +} +trap cleanup EXIT -chmod ugo+w ${DB_PATH} +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table1' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table2' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table3' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table4' +sqlite3 "${DB_PATH}" 'DROP TABLE IF EXISTS table5' -sqlite3 ${DB_PATH} "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" -sqlite3 ${DB_PATH} "INSERT INTO table2 VALUES (1, 'text1'), (2, 'text2'), (3, 'text3')" +sqlite3 "${DB_PATH}" 'CREATE TABLE table1 (col1 text, col2 smallint);' +sqlite3 "${DB_PATH}" 'CREATE TABLE table2 (col1 int, col2 text);' -sqlite3 ${DB_PATH} 'CREATE TABLE table3 (col1 text, col2 int);' -sqlite3 ${DB_PATH} 'INSERT INTO table3 VALUES (NULL, 1)' -sqlite3 ${DB_PATH} "INSERT INTO table3 VALUES ('not a null', 2)" -sqlite3 ${DB_PATH} 'INSERT INTO table3 VALUES (NULL, 3)' -sqlite3 ${DB_PATH} "INSERT INTO table3 VALUES ('', 4)" +chmod ugo+w "${DB_PATH}" -sqlite3 ${DB_PATH} 'CREATE TABLE table4 (a int, b integer, c tinyint, d smallint, e mediumint, bigint, int2, int8)' -sqlite3 ${DB_PATH} 'CREATE TABLE table5 (a character(20), b varchar(10), c real, d double, e double precision, f float)' +sqlite3 "${DB_PATH}" "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" +sqlite3 "${DB_PATH}" "INSERT INTO table2 VALUES (1, 'text1'), (2, 'text2'), (3, 'text3')" +sqlite3 "${DB_PATH}" 'CREATE TABLE table3 (col1 text, col2 int);' +sqlite3 "${DB_PATH}" 'INSERT INTO table3 VALUES (NULL, 1)' +sqlite3 "${DB_PATH}" "INSERT INTO table3 VALUES ('not a null', 2)" +sqlite3 "${DB_PATH}" 'INSERT INTO table3 VALUES (NULL, 3)' +sqlite3 "${DB_PATH}" "INSERT INTO table3 VALUES ('', 4)" + +sqlite3 "${DB_PATH}" 'CREATE TABLE table4 (a int, b integer, c tinyint, d smallint, e mediumint, bigint, int2, int8)' +sqlite3 "${DB_PATH}" 'CREATE TABLE table5 (a character(20), b varchar(10), c real, d double, e double precision, f float)' -${CLICKHOUSE_CLIENT} --query='DROP DATABASE IF EXISTS sqlite_database' ${CLICKHOUSE_CLIENT} --query="select 'create database engine'"; -${CLICKHOUSE_CLIENT} --query="CREATE DATABASE sqlite_database ENGINE = SQLite('${DB_PATH}')" +${CLICKHOUSE_CLIENT} --query="CREATE DATABASE ${CURR_DATABASE} ENGINE = SQLite('${DB_PATH}')" ${CLICKHOUSE_CLIENT} --query="select 'show database tables:'"; -${CLICKHOUSE_CLIENT} --query='SHOW TABLES FROM sqlite_database;' +${CLICKHOUSE_CLIENT} --query='SHOW TABLES FROM '"${CURR_DATABASE}"';' ${CLICKHOUSE_CLIENT} --query="select 'show creare table:'"; -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table1;' | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table2;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table1;" | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table2;" | sed -r 's/(.*SQLite)(.*)/\1/' ${CLICKHOUSE_CLIENT} --query="select 'describe table:'"; -${CLICKHOUSE_CLIENT} --query='DESCRIBE TABLE sqlite_database.table1;' -${CLICKHOUSE_CLIENT} --query='DESCRIBE TABLE sqlite_database.table2;' +${CLICKHOUSE_CLIENT} --query="DESCRIBE TABLE ${CURR_DATABASE}.table1;" +${CLICKHOUSE_CLIENT} --query="DESCRIBE TABLE ${CURR_DATABASE}.table2;" ${CLICKHOUSE_CLIENT} --query="select 'select *:'"; -${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_database.table1 ORDER BY col2' -${CLICKHOUSE_CLIENT} --query='SELECT * FROM sqlite_database.table2 ORDER BY col1;' +${CLICKHOUSE_CLIENT} --query="SELECT * FROM ${CURR_DATABASE}.table1 ORDER BY col2" +${CLICKHOUSE_CLIENT} --query="SELECT * FROM ${CURR_DATABASE}.table2 ORDER BY col1" ${CLICKHOUSE_CLIENT} --query="select 'test types'"; -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table4;' | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='SHOW CREATE TABLE sqlite_database.table5;' | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table4;" | sed -r 's/(.*SQLite)(.*)/\1/' +${CLICKHOUSE_CLIENT} --query="SHOW CREATE TABLE ${CURR_DATABASE}.table5;" | sed -r 's/(.*SQLite)(.*)/\1/' -${CLICKHOUSE_CLIENT} --query='DROP DATABASE IF EXISTS sqlite_database' +${CLICKHOUSE_CLIENT} --query="DROP DATABASE IF EXISTS ${CURR_DATABASE}" ${CLICKHOUSE_CLIENT} --query="select 'create table engine with table3'"; @@ -79,11 +87,9 @@ ${CLICKHOUSE_CLIENT} --query="INSERT INTO TABLE FUNCTION sqlite('${DB_PATH}', 't ${CLICKHOUSE_CLIENT} --query="SELECT * FROM sqlite('${DB_PATH}', 'table1') ORDER BY col2" -sqlite3 $CUR_DIR/db2 'DROP TABLE IF EXISTS table1' -sqlite3 $CUR_DIR/db2 'CREATE TABLE table1 (col1 text, col2 smallint);' -sqlite3 $CUR_DIR/db2 "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" +sqlite3 "${DB_PATH2}" 'DROP TABLE IF EXISTS table1' +sqlite3 "${DB_PATH2}" 'CREATE TABLE table1 (col1 text, col2 smallint);' +sqlite3 "${DB_PATH2}" "INSERT INTO table1 VALUES ('line1', 1), ('line2', 2), ('line3', 3)" ${CLICKHOUSE_CLIENT} --query="select 'test path in clickhouse-local'"; -${CLICKHOUSE_LOCAL} --query="SELECT * FROM sqlite('$CUR_DIR/db2', 'table1') ORDER BY col2" - -rm -r ${DB_PATH} +${CLICKHOUSE_LOCAL} --query="SELECT * FROM sqlite('${DB_PATH2}', 'table1') ORDER BY col2" diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index 83ad14c44dc..c24de285856 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -487,7 +487,6 @@ "01824_prefer_global_in_and_join", "01870_modulo_partition_key", "01870_buffer_flush", // creates database - "01889_sqlite_read_write", "01889_postgresql_protocol_null_fields", "01889_check_row_policy_defined_using_user_function", "01921_concurrent_ttl_and_normal_merges_zookeeper_long", // heavy test, better to run sequentially From 0e5cfdbb9dbe0f6d823a99be508b4cd6c824444c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:27:51 +0200 Subject: [PATCH 137/220] 01054_cache_dictionary_overflow_cell: Drop database at the end --- .../queries/0_stateless/01054_cache_dictionary_overflow_cell.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql b/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql index d8d1d61be63..1b317e2165e 100644 --- a/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql +++ b/tests/queries/0_stateless/01054_cache_dictionary_overflow_cell.sql @@ -54,3 +54,4 @@ FROM ); DROP TABLE if exists test_01054.ints; +DROP DATABASE test_01054_overflow; From 19f087cecefb2cd5e4c8e9569ea224a7659e9fe6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:31:40 +0200 Subject: [PATCH 138/220] 01114_mysql_database_engine_segfault: Cleanup beforehand --- .../queries/0_stateless/01114_mysql_database_engine_segfault.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql index af88c5af53a..5893365e11c 100644 --- a/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql +++ b/tests/queries/0_stateless/01114_mysql_database_engine_segfault.sql @@ -1 +1,2 @@ +DROP DATABASE IF EXISTS conv_main; CREATE DATABASE conv_main ENGINE = MySQL('127.0.0.1:3456', conv_main, 'metrika', 'password'); -- { serverError 501 } From b80ddd4bd1f64cbb634583fda44d32a1303211d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:41:31 +0200 Subject: [PATCH 139/220] 01516_drop_table_stress: Parallelizable and cleanup --- .../0_stateless/01516_drop_table_stress.sh | 21 +++++++++++-------- tests/queries/skip_list.json | 1 - 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/tests/queries/0_stateless/01516_drop_table_stress.sh b/tests/queries/0_stateless/01516_drop_table_stress.sh index d72104c8c7f..20e76ce49c7 100755 --- a/tests/queries/0_stateless/01516_drop_table_stress.sh +++ b/tests/queries/0_stateless/01516_drop_table_stress.sh @@ -4,26 +4,29 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh +export CURR_DATABASE="test_01516_${CLICKHOUSE_DATABASE}" + function drop_database() { # redirect stderr since it is racy with DROP TABLE - # and tries to remove db_01516.data too. - ${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS db_01516" 2>/dev/null + # and tries to remove ${CURR_DATABASE}.data too. + ${CLICKHOUSE_CLIENT} -q "DROP DATABASE IF EXISTS ${CURR_DATABASE}" 2>/dev/null } +trap drop_database EXIT function drop_table() { - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data3;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data1;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS db_01516.data2;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data3;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data1;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS ${CURR_DATABASE}.data2;" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" } function create() { - ${CLICKHOUSE_CLIENT} -q "CREATE DATABASE IF NOT EXISTS db_01516;" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data1 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data2 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" - ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS db_01516.data3 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE DATABASE IF NOT EXISTS ${CURR_DATABASE};" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data1 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data2 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" + ${CLICKHOUSE_CLIENT} -q "CREATE TABLE IF NOT EXISTS ${CURR_DATABASE}.data3 Engine=MergeTree() ORDER BY number AS SELECT * FROM numbers(1);" 2>&1 | grep -F "Code: " | grep -Fv "is currently dropped or renamed" } for _ in {1..100}; do diff --git a/tests/queries/skip_list.json b/tests/queries/skip_list.json index c24de285856..c2b5782e766 100644 --- a/tests/queries/skip_list.json +++ b/tests/queries/skip_list.json @@ -416,7 +416,6 @@ "01507_clickhouse_server_start_with_embedded_config", "01509_dictionary_preallocate", "01516_create_table_primary_key", - "01516_drop_table_stress", "01517_drop_mv_with_inner_table", "01526_complex_key_dict_direct_layout", "01527_clickhouse_local_optimize", From f7ed6c4fb8fa7b27d67a8967298cfd9bd88cc66b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:42:31 +0200 Subject: [PATCH 140/220] 01516_drop_table_stress: Tag as long --- ...le_stress.reference => 01516_drop_table_stress_long.reference} | 0 ...01516_drop_table_stress.sh => 01516_drop_table_stress_long.sh} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename tests/queries/0_stateless/{01516_drop_table_stress.reference => 01516_drop_table_stress_long.reference} (100%) rename tests/queries/0_stateless/{01516_drop_table_stress.sh => 01516_drop_table_stress_long.sh} (100%) diff --git a/tests/queries/0_stateless/01516_drop_table_stress.reference b/tests/queries/0_stateless/01516_drop_table_stress_long.reference similarity index 100% rename from tests/queries/0_stateless/01516_drop_table_stress.reference rename to tests/queries/0_stateless/01516_drop_table_stress_long.reference diff --git a/tests/queries/0_stateless/01516_drop_table_stress.sh b/tests/queries/0_stateless/01516_drop_table_stress_long.sh similarity index 100% rename from tests/queries/0_stateless/01516_drop_table_stress.sh rename to tests/queries/0_stateless/01516_drop_table_stress_long.sh From ac5ac0a106a6c2217210bb358e3d8e97ed3a898d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 17 Aug 2021 18:48:32 +0200 Subject: [PATCH 141/220] 01280_ssd_complex_key_dictionary: Delete database at end --- tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh index ff62b70c184..e1e0018a1dd 100755 --- a/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh +++ b/tests/queries/0_stateless/01280_ssd_complex_key_dictionary.sh @@ -122,3 +122,5 @@ $CLICKHOUSE_CLIENT -n --query="DROP DICTIONARY 01280_db.ssd_dict; SELECT arrayJoin([('1', toInt32(3)), ('2', toInt32(-1)), ('', toInt32(0)), ('', toInt32(0)), ('2', toInt32(-1)), ('1', toInt32(3))]) AS keys, dictGetInt32('01280_db.ssd_dict', 'b', keys); DROP DICTIONARY IF EXISTS database_for_dict.ssd_dict; DROP TABLE IF EXISTS database_for_dict.keys_table;" + +$CLICKHOUSE_CLIENT -n --query="DROP DATABASE IF EXISTS 01280_db;" From b4f41bd8247a47b6300ebcdc02849225ca501d64 Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 17 Aug 2021 20:35:43 +0300 Subject: [PATCH 142/220] Dictionaries key types refactoring --- src/Dictionaries/CacheDictionary.cpp | 20 +-- src/Dictionaries/CacheDictionary.h | 13 +- src/Dictionaries/CacheDictionaryStorage.h | 25 ++- .../CacheDictionaryUpdateQueue.cpp | 8 +- src/Dictionaries/CacheDictionaryUpdateQueue.h | 11 +- src/Dictionaries/DictionaryHelpers.h | 16 +- src/Dictionaries/DictionarySource.cpp | 2 +- src/Dictionaries/DirectDictionary.cpp | 22 +-- src/Dictionaries/DirectDictionary.h | 9 +- src/Dictionaries/FlatDictionary.cpp | 6 +- src/Dictionaries/FlatDictionary.h | 2 +- src/Dictionaries/HashedDictionary.cpp | 42 ++--- src/Dictionaries/HashedDictionary.h | 23 ++- src/Dictionaries/IDictionary.h | 20 ++- src/Dictionaries/IPAddressDictionary.h | 2 +- src/Dictionaries/PolygonDictionary.h | 2 +- src/Dictionaries/RangeDictionarySource.h | 73 ++------ src/Dictionaries/RangeHashedDictionary.cpp | 26 ++- src/Dictionaries/RangeHashedDictionary.h | 9 +- src/Dictionaries/SSDCacheDictionaryStorage.h | 32 ++-- .../registerCacheDictionaries.cpp | 23 ++- src/Functions/FunctionsExternalDictionaries.h | 159 +++++++----------- 22 files changed, 238 insertions(+), 307 deletions(-) diff --git a/src/Dictionaries/CacheDictionary.cpp b/src/Dictionaries/CacheDictionary.cpp index a5f953ccc15..0b2044cfe2c 100644 --- a/src/Dictionaries/CacheDictionary.cpp +++ b/src/Dictionaries/CacheDictionary.cpp @@ -151,7 +151,7 @@ Columns CacheDictionary::getColumns( * use default value. */ - if (dictionary_key_type == DictionaryKeyType::complex) + if (dictionary_key_type == DictionaryKeyType::Complex) dict_struct.validateKeyTypes(key_types); DictionaryKeysArenaHolder arena_holder; @@ -268,7 +268,7 @@ ColumnUInt8::Ptr CacheDictionary::hasKeys(const Columns & k * Check that key was fetched during update for that key set true in result array. */ - if (dictionary_key_type == DictionaryKeyType::complex) + if (dictionary_key_type == DictionaryKeyType::Complex) dict_struct.validateKeyTypes(key_types); @@ -364,7 +364,7 @@ ColumnPtr CacheDictionary::getHierarchy( ColumnPtr key_column [[maybe_unused]], const DataTypePtr & key_type [[maybe_unused]]) const { - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) { size_t keys_found; auto result = getKeysHierarchyDefaultImplementation(this, key_column, key_type, keys_found); @@ -382,7 +382,7 @@ ColumnUInt8::Ptr CacheDictionary::isInHierarchy( ColumnPtr in_key_column [[maybe_unused]], const DataTypePtr & key_type [[maybe_unused]]) const { - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) { size_t keys_found; auto result = getKeysIsInHierarchyDefaultImplementation(this, key_column, in_key_column, key_type, keys_found); @@ -492,7 +492,7 @@ Pipe CacheDictionary::read(const Names & column_names, size /// Write lock on storage const ProfilingScopedWriteRWLock write_lock{rw_lock, ProfileEvents::DictCacheLockWriteNs}; - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) data.emplace(shared_from_this(), cache_storage_ptr->getCachedSimpleKeys(), column_names); else { @@ -534,7 +534,7 @@ void CacheDictionary::update(CacheDictionaryUpdateUnitPtr requested_keys_vector; std::vector requested_complex_key_rows; - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) requested_keys_vector.reserve(requested_keys.size()); else requested_complex_key_rows.reserve(requested_keys.size()); @@ -546,7 +546,7 @@ void CacheDictionary::update(CacheDictionaryUpdateUnitPtr::update(CacheDictionaryUpdateUnitPtrloadIds(requested_keys_vector)); else pipeline.init(current_source_ptr->loadKeys(update_unit_ptr->key_columns, requested_complex_key_rows)); @@ -684,7 +684,7 @@ void CacheDictionary::update(CacheDictionaryUpdateUnitPtr; -template class CacheDictionary; +template class CacheDictionary; +template class CacheDictionary; } diff --git a/src/Dictionaries/CacheDictionary.h b/src/Dictionaries/CacheDictionary.h index 613d73b0f83..0e2a2699c45 100644 --- a/src/Dictionaries/CacheDictionary.h +++ b/src/Dictionaries/CacheDictionary.h @@ -51,8 +51,7 @@ template class CacheDictionary final : public IDictionary { public: - using KeyType = std::conditional_t; - static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by cache dictionary"); + using KeyType = std::conditional_t; CacheDictionary( const StorageID & dict_id_, @@ -118,7 +117,7 @@ public: DictionaryKeyType getKeyType() const override { - return dictionary_key_type == DictionaryKeyType::simple ? DictionaryKeyType::simple : DictionaryKeyType::complex; + return dictionary_key_type == DictionaryKeyType::Simple ? DictionaryKeyType::Simple : DictionaryKeyType::Complex; } ColumnPtr getColumn( @@ -141,7 +140,7 @@ public: std::exception_ptr getLastException() const override; - bool hasHierarchy() const override { return dictionary_key_type == DictionaryKeyType::simple && dict_struct.hierarchical_attribute_index.has_value(); } + bool hasHierarchy() const override { return dictionary_key_type == DictionaryKeyType::Simple && dict_struct.hierarchical_attribute_index.has_value(); } ColumnPtr getHierarchy(ColumnPtr key_column, const DataTypePtr & key_type) const override; @@ -151,7 +150,7 @@ public: const DataTypePtr & key_type) const override; private: - using FetchResult = std::conditional_t; + using FetchResult = std::conditional_t; static MutableColumns aggregateColumnsInOrderOfKeys( const PaddedPODArray & keys, @@ -219,7 +218,7 @@ private: }; -extern template class CacheDictionary; -extern template class CacheDictionary; +extern template class CacheDictionary; +extern template class CacheDictionary; } diff --git a/src/Dictionaries/CacheDictionaryStorage.h b/src/Dictionaries/CacheDictionaryStorage.h index 8374e649cd1..2c7e9ad7092 100644 --- a/src/Dictionaries/CacheDictionaryStorage.h +++ b/src/Dictionaries/CacheDictionaryStorage.h @@ -41,8 +41,7 @@ class CacheDictionaryStorage final : public ICacheDictionaryStorage static constexpr size_t max_collision_length = 10; public: - using KeyType = std::conditional_t; - static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by CacheDictionaryStorage"); + using KeyType = std::conditional_t; explicit CacheDictionaryStorage( const DictionaryStructure & dictionary_structure, @@ -62,19 +61,19 @@ public: String getName() const override { - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) return "Cache"; else return "ComplexKeyCache"; } - bool supportsSimpleKeys() const override { return dictionary_key_type == DictionaryKeyType::simple; } + bool supportsSimpleKeys() const override { return dictionary_key_type == DictionaryKeyType::Simple; } SimpleKeysStorageFetchResult fetchColumnsForKeys( const PaddedPODArray & keys, const DictionaryStorageFetchRequest & fetch_request) override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) return fetchColumnsForKeysImpl(keys, fetch_request); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method fetchColumnsForKeys is not supported for complex key storage"); @@ -82,7 +81,7 @@ public: void insertColumnsForKeys(const PaddedPODArray & keys, Columns columns) override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) insertColumnsForKeysImpl(keys, columns); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertColumnsForKeys is not supported for complex key storage"); @@ -90,7 +89,7 @@ public: void insertDefaultKeys(const PaddedPODArray & keys) override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) insertDefaultKeysImpl(keys); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertDefaultKeysImpl is not supported for complex key storage"); @@ -98,19 +97,19 @@ public: PaddedPODArray getCachedSimpleKeys() const override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) return getCachedKeysImpl(); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getCachedSimpleKeys is not supported for complex key storage"); } - bool supportsComplexKeys() const override { return dictionary_key_type == DictionaryKeyType::complex; } + bool supportsComplexKeys() const override { return dictionary_key_type == DictionaryKeyType::Complex; } ComplexKeysStorageFetchResult fetchColumnsForKeys( const PaddedPODArray & keys, const DictionaryStorageFetchRequest & column_fetch_requests) override { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) return fetchColumnsForKeysImpl(keys, column_fetch_requests); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method fetchColumnsForKeys is not supported for simple key storage"); @@ -118,7 +117,7 @@ public: void insertColumnsForKeys(const PaddedPODArray & keys, Columns columns) override { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) insertColumnsForKeysImpl(keys, columns); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertColumnsForKeys is not supported for simple key storage"); @@ -126,7 +125,7 @@ public: void insertDefaultKeys(const PaddedPODArray & keys) override { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) insertDefaultKeysImpl(keys); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertDefaultKeysImpl is not supported for simple key storage"); @@ -134,7 +133,7 @@ public: PaddedPODArray getCachedComplexKeys() const override { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) return getCachedKeysImpl(); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getCachedComplexKeys is not supported for simple key storage"); diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp index 310abed822f..1d96fcc108b 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.cpp +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.cpp @@ -14,8 +14,8 @@ namespace ErrorCodes extern const int TIMEOUT_EXCEEDED; } -template class CacheDictionaryUpdateUnit; -template class CacheDictionaryUpdateUnit; +template class CacheDictionaryUpdateUnit; +template class CacheDictionaryUpdateUnit; template CacheDictionaryUpdateQueue::CacheDictionaryUpdateQueue( @@ -155,7 +155,7 @@ void CacheDictionaryUpdateQueue::updateThreadFunction() } } -template class CacheDictionaryUpdateQueue; -template class CacheDictionaryUpdateQueue; +template class CacheDictionaryUpdateQueue; +template class CacheDictionaryUpdateQueue; } diff --git a/src/Dictionaries/CacheDictionaryUpdateQueue.h b/src/Dictionaries/CacheDictionaryUpdateQueue.h index 3d27a157752..bcad376bc53 100644 --- a/src/Dictionaries/CacheDictionaryUpdateQueue.h +++ b/src/Dictionaries/CacheDictionaryUpdateQueue.h @@ -39,7 +39,7 @@ template class CacheDictionaryUpdateUnit { public: - using KeyType = std::conditional_t; + using KeyType = std::conditional_t; /// Constructor for complex keys update request explicit CacheDictionaryUpdateUnit( @@ -85,8 +85,8 @@ private: template using CacheDictionaryUpdateUnitPtr = std::shared_ptr>; -extern template class CacheDictionaryUpdateUnit; -extern template class CacheDictionaryUpdateUnit; +extern template class CacheDictionaryUpdateUnit; +extern template class CacheDictionaryUpdateUnit; struct CacheDictionaryUpdateQueueConfiguration { @@ -110,7 +110,6 @@ class CacheDictionaryUpdateQueue public: /// Client of update queue must provide this function in constructor and perform update using update unit. using UpdateFunction = std::function)>; - static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by CacheDictionaryUpdateQueue"); CacheDictionaryUpdateQueue( String dictionary_name_for_logs_, @@ -167,7 +166,7 @@ private: std::atomic finished{false}; }; -extern template class CacheDictionaryUpdateQueue; -extern template class CacheDictionaryUpdateQueue; +extern template class CacheDictionaryUpdateQueue; +extern template class CacheDictionaryUpdateQueue; } diff --git a/src/Dictionaries/DictionaryHelpers.h b/src/Dictionaries/DictionaryHelpers.h index 3d077414291..0c04d87c959 100644 --- a/src/Dictionaries/DictionaryHelpers.h +++ b/src/Dictionaries/DictionaryHelpers.h @@ -380,14 +380,14 @@ template class DictionaryKeysArenaHolder; template <> -class DictionaryKeysArenaHolder +class DictionaryKeysArenaHolder { public: static Arena * getComplexKeyArena() { return nullptr; } }; template <> -class DictionaryKeysArenaHolder +class DictionaryKeysArenaHolder { public: @@ -402,8 +402,7 @@ template class DictionaryKeysExtractor { public: - using KeyType = std::conditional_t; - static_assert(key_type != DictionaryKeyType::range, "Range key type is not supported by DictionaryKeysExtractor"); + using KeyType = std::conditional_t; explicit DictionaryKeysExtractor(const Columns & key_columns_, Arena * complex_key_arena_) : key_columns(key_columns_) @@ -411,7 +410,7 @@ public: { assert(!key_columns.empty()); - if constexpr (key_type == DictionaryKeyType::simple) + if constexpr (key_type == DictionaryKeyType::Simple) { key_columns[0] = key_columns[0]->convertToFullColumnIfConst(); @@ -437,7 +436,7 @@ public: { assert(current_key_index < keys_size); - if constexpr (key_type == DictionaryKeyType::simple) + if constexpr (key_type == DictionaryKeyType::Simple) { const auto & column_vector = static_cast &>(*key_columns[0]); const auto & data = column_vector.getData(); @@ -465,7 +464,7 @@ public: void rollbackCurrentKey() const { - if constexpr (key_type == DictionaryKeyType::complex) + if constexpr (key_type == DictionaryKeyType::Complex) complex_key_arena->rollback(current_complex_key.size); } @@ -521,8 +520,7 @@ void mergeBlockWithPipe( Block & block_to_update, Pipe pipe) { - using KeyType = std::conditional_t; - static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by updatePreviousyLoadedBlockWithStream"); + using KeyType = std::conditional_t; Columns saved_block_key_columns; saved_block_key_columns.reserve(key_columns_size); diff --git a/src/Dictionaries/DictionarySource.cpp b/src/Dictionaries/DictionarySource.cpp index fbb03cb00fa..a164543e1ff 100644 --- a/src/Dictionaries/DictionarySource.cpp +++ b/src/Dictionaries/DictionarySource.cpp @@ -132,7 +132,7 @@ Block DictionarySourceData::fillBlock( { ColumnPtr column; - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) { column = dictionary->getColumn( attribute.name, diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index eb06701ab7a..e12100a556d 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -40,7 +40,7 @@ Columns DirectDictionary::getColumns( const DataTypes & key_types [[maybe_unused]], const Columns & default_values_columns) const { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) dict_struct.validateKeyTypes(key_types); DictionaryKeysArenaHolder arena_holder; @@ -161,7 +161,7 @@ ColumnUInt8::Ptr DirectDictionary::hasKeys( const Columns & key_columns, const DataTypes & key_types [[maybe_unused]]) const { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) dict_struct.validateKeyTypes(key_types); DictionaryKeysArenaHolder arena_holder; @@ -230,7 +230,7 @@ ColumnPtr DirectDictionary::getHierarchy( ColumnPtr key_column, const DataTypePtr & key_type) const { - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) { size_t keys_found; auto result = getKeysHierarchyDefaultImplementation(this, key_column, key_type, keys_found); @@ -248,7 +248,7 @@ ColumnUInt8::Ptr DirectDictionary::isInHierarchy( ColumnPtr in_key_column, const DataTypePtr & key_type) const { - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) { size_t keys_found = 0; auto result = getKeysIsInHierarchyDefaultImplementation(this, key_column, in_key_column, key_type, keys_found); @@ -269,7 +269,7 @@ Pipe DirectDictionary::getSourceBlockInputStream( Pipe pipe; - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) { std::vector ids; ids.reserve(requested_keys_size); @@ -310,9 +310,9 @@ namespace ContextPtr /* context */, bool /* created_from_ddl */) { - const auto * layout_name = dictionary_key_type == DictionaryKeyType::simple ? "direct" : "complex_key_direct"; + const auto * layout_name = dictionary_key_type == DictionaryKeyType::Simple ? "direct" : "complex_key_direct"; - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) { if (dict_struct.key) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, @@ -344,13 +344,13 @@ namespace } } -template class DirectDictionary; -template class DirectDictionary; +template class DirectDictionary; +template class DirectDictionary; void registerDictionaryDirect(DictionaryFactory & factory) { - factory.registerLayout("direct", createDirectDictionary, false); - factory.registerLayout("complex_key_direct", createDirectDictionary, true); + factory.registerLayout("direct", createDirectDictionary, false); + factory.registerLayout("complex_key_direct", createDirectDictionary, true); } diff --git a/src/Dictionaries/DirectDictionary.h b/src/Dictionaries/DirectDictionary.h index 4700e71d94b..ebe5f5fbbc7 100644 --- a/src/Dictionaries/DirectDictionary.h +++ b/src/Dictionaries/DirectDictionary.h @@ -20,8 +20,7 @@ template class DirectDictionary final : public IDictionary { public: - static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by direct dictionary"); - using KeyType = std::conditional_t; + using KeyType = std::conditional_t; DirectDictionary( const StorageID & dict_id_, @@ -30,7 +29,7 @@ public: std::string getTypeName() const override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) return "Direct"; else return "ComplexKeyDirect"; @@ -110,7 +109,7 @@ private: mutable std::atomic found_count{0}; }; -extern template class DirectDictionary; -extern template class DirectDictionary; +extern template class DirectDictionary; +extern template class DirectDictionary; } diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index 639895ac8ac..26667db1081 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -289,8 +289,8 @@ void FlatDictionary::blockToAttributes(const Block & block) { const auto keys_column = block.safeGetByPosition(0).column; - DictionaryKeysArenaHolder arena_holder; - DictionaryKeysExtractor keys_extractor({ keys_column }, arena_holder.getComplexKeyArena()); + DictionaryKeysArenaHolder arena_holder; + DictionaryKeysExtractor keys_extractor({ keys_column }, arena_holder.getComplexKeyArena()); auto keys = keys_extractor.extractAllKeys(); HashSet already_processed_keys; @@ -344,7 +344,7 @@ void FlatDictionary::updateData() else { Pipe pipe(source_ptr->loadUpdatedAll()); - mergeBlockWithPipe( + mergeBlockWithPipe( dict_struct.getKeysSize(), *update_field_loaded_block, std::move(pipe)); diff --git a/src/Dictionaries/FlatDictionary.h b/src/Dictionaries/FlatDictionary.h index e6a07200c05..196194ddb21 100644 --- a/src/Dictionaries/FlatDictionary.h +++ b/src/Dictionaries/FlatDictionary.h @@ -72,7 +72,7 @@ public: return dict_struct.getAttribute(attribute_name).injective; } - DictionaryKeyType getKeyType() const override { return DictionaryKeyType::simple; } + DictionaryKeyType getKeyType() const override { return DictionaryKeyType::Simple; } ColumnPtr getColumn( const std::string& attribute_name, diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index 189994dabf4..d462631fba8 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -61,7 +61,7 @@ ColumnPtr HashedDictionary::getColumn( const DataTypes & key_types [[maybe_unused]], const ColumnPtr & default_values_column) const { - if (dictionary_key_type == DictionaryKeyType::complex) + if (dictionary_key_type == DictionaryKeyType::Complex) dict_struct.validateKeyTypes(key_types); ColumnPtr result; @@ -163,7 +163,7 @@ ColumnPtr HashedDictionary::getColumn( template ColumnUInt8::Ptr HashedDictionary::hasKeys(const Columns & key_columns, const DataTypes & key_types) const { - if (dictionary_key_type == DictionaryKeyType::complex) + if (dictionary_key_type == DictionaryKeyType::Complex) dict_struct.validateKeyTypes(key_types); DictionaryKeysArenaHolder arena_holder; @@ -210,7 +210,7 @@ ColumnUInt8::Ptr HashedDictionary::hasKeys(const Co template ColumnPtr HashedDictionary::getHierarchy(ColumnPtr key_column [[maybe_unused]], const DataTypePtr &) const { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) { PaddedPODArray keys_backup_storage; const auto & keys = getColumnVectorData(this, key_column, keys_backup_storage); @@ -258,7 +258,7 @@ ColumnUInt8::Ptr HashedDictionary::isInHierarchy( ColumnPtr in_key_column [[maybe_unused]], const DataTypePtr &) const { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) { PaddedPODArray keys_backup_storage; const auto & keys = getColumnVectorData(this, key_column, keys_backup_storage); @@ -309,7 +309,7 @@ ColumnPtr HashedDictionary::getDescendants( const DataTypePtr &, size_t level [[maybe_unused]]) const { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) { PaddedPODArray keys_backup; const auto & keys = getColumnVectorData(this, key_column, keys_backup); @@ -665,7 +665,7 @@ Pipe HashedDictionary::read(const Names & column_na }); } - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) return Pipe(std::make_shared(DictionarySourceData(shared_from_this(), std::move(keys), column_names), max_block_size)); else return Pipe(std::make_shared(DictionarySourceData(shared_from_this(), keys, column_names), max_block_size)); @@ -702,10 +702,10 @@ void HashedDictionary::getAttributeContainer(size_t }); } -template class HashedDictionary; -template class HashedDictionary; -template class HashedDictionary; -template class HashedDictionary; +template class HashedDictionary; +template class HashedDictionary; +template class HashedDictionary; +template class HashedDictionary; void registerDictionaryHashed(DictionaryFactory & factory) { @@ -717,9 +717,9 @@ void registerDictionaryHashed(DictionaryFactory & factory) DictionaryKeyType dictionary_key_type, bool sparse) -> DictionaryPtr { - if (dictionary_key_type == DictionaryKeyType::simple && dict_struct.key) + if (dictionary_key_type == DictionaryKeyType::Simple && dict_struct.key) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "'key' is not supported for simple key hashed dictionary"); - else if (dictionary_key_type == DictionaryKeyType::complex && dict_struct.id) + else if (dictionary_key_type == DictionaryKeyType::Complex && dict_struct.id) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "'id' is not supported for complex key hashed dictionary"); if (dict_struct.range_min || dict_struct.range_max) @@ -737,32 +737,32 @@ void registerDictionaryHashed(DictionaryFactory & factory) HashedDictionaryStorageConfiguration configuration{preallocate, require_nonempty, dict_lifetime}; - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) { if (sparse) - return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), configuration); + return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), configuration); else - return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), configuration); + return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), configuration); } else { if (sparse) - return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), configuration); + return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), configuration); else - return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), configuration); + return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), configuration); } }; using namespace std::placeholders; factory.registerLayout("hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::simple, /* sparse = */ false); }, false); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ false); }, false); factory.registerLayout("sparse_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::simple, /* sparse = */ true); }, false); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ true); }, false); factory.registerLayout("complex_key_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::complex, /* sparse = */ false); }, true); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ false); }, true); factory.registerLayout("complex_key_sparse_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::complex, /* sparse = */ true); }, true); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ true); }, true); } diff --git a/src/Dictionaries/HashedDictionary.h b/src/Dictionaries/HashedDictionary.h index bf58638effc..d1e1f681fa1 100644 --- a/src/Dictionaries/HashedDictionary.h +++ b/src/Dictionaries/HashedDictionary.h @@ -35,8 +35,7 @@ template class HashedDictionary final : public IDictionary { public: - using KeyType = std::conditional_t; - static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by hashed dictionary"); + using KeyType = std::conditional_t; HashedDictionary( const StorageID & dict_id_, @@ -47,11 +46,11 @@ public: std::string getTypeName() const override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple && sparse) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple && sparse) return "SparseHashed"; - else if constexpr (dictionary_key_type == DictionaryKeyType::simple && !sparse) + else if constexpr (dictionary_key_type == DictionaryKeyType::Simple && !sparse) return "Hashed"; - else if constexpr (dictionary_key_type == DictionaryKeyType::complex && sparse) + else if constexpr (dictionary_key_type == DictionaryKeyType::Complex && sparse) return "ComplexKeySparseHashed"; else return "ComplexKeyHashed"; @@ -102,7 +101,7 @@ public: ColumnUInt8::Ptr hasKeys(const Columns & key_columns, const DataTypes & key_types) const override; - bool hasHierarchy() const override { return dictionary_key_type == DictionaryKeyType::simple && dict_struct.hierarchical_attribute_index.has_value(); } + bool hasHierarchy() const override { return dictionary_key_type == DictionaryKeyType::Simple && dict_struct.hierarchical_attribute_index.has_value(); } ColumnPtr getHierarchy(ColumnPtr key_column, const DataTypePtr & hierarchy_attribute_type) const override; @@ -121,13 +120,13 @@ public: private: template using CollectionTypeNonSparse = std::conditional_t< - dictionary_key_type == DictionaryKeyType::simple, + dictionary_key_type == DictionaryKeyType::Simple, HashMap, HashMapWithSavedHash>>; template using CollectionTypeSparse = std::conditional_t< - dictionary_key_type == DictionaryKeyType::simple, + dictionary_key_type == DictionaryKeyType::Simple, SparseHashMap, SparseHashMap>; @@ -211,10 +210,10 @@ private: Arena complex_key_arena; }; -extern template class HashedDictionary; -extern template class HashedDictionary; +extern template class HashedDictionary; +extern template class HashedDictionary; -extern template class HashedDictionary; -extern template class HashedDictionary; +extern template class HashedDictionary; +extern template class HashedDictionary; } diff --git a/src/Dictionaries/IDictionary.h b/src/Dictionaries/IDictionary.h index f9e0223a698..d7778cc0022 100644 --- a/src/Dictionaries/IDictionary.h +++ b/src/Dictionaries/IDictionary.h @@ -33,15 +33,20 @@ using DictionaryPtr = std::unique_ptr; * Simple is for dictionaries that support UInt64 key column. * * Complex is for dictionaries that support any combination of key columns. - * - * Range is for dictionary that support combination of UInt64 key column, - * and numeric representable range key column. */ enum class DictionaryKeyType { - simple, - complex, - range + Simple, + Complex +}; + +/** DictionarySpecialKeyType provides IDictionary client information about + * which special key type is supported by dictionary. + */ +enum class DictionarySpecialKeyType +{ + None, + Range }; /** @@ -56,6 +61,7 @@ struct IDictionary : public IExternalLoadable } const std::string & getFullName() const{ return full_name; } + StorageID getDictionaryID() const { std::lock_guard lock{name_mutex}; @@ -109,6 +115,8 @@ struct IDictionary : public IExternalLoadable */ virtual DictionaryKeyType getKeyType() const = 0; + virtual DictionarySpecialKeyType getSpecialKeyType() const { return DictionarySpecialKeyType::None;} + /** Subclass must validate key columns and keys types * and return column representation of dictionary attribute. * diff --git a/src/Dictionaries/IPAddressDictionary.h b/src/Dictionaries/IPAddressDictionary.h index af4b77a6ff8..4aba70dd14f 100644 --- a/src/Dictionaries/IPAddressDictionary.h +++ b/src/Dictionaries/IPAddressDictionary.h @@ -67,7 +67,7 @@ public: return dict_struct.getAttribute(attribute_name).injective; } - DictionaryKeyType getKeyType() const override { return DictionaryKeyType::complex; } + DictionaryKeyType getKeyType() const override { return DictionaryKeyType::Complex; } ColumnPtr getColumn( const std::string& attribute_name, diff --git a/src/Dictionaries/PolygonDictionary.h b/src/Dictionaries/PolygonDictionary.h index 32771be4b16..9ab82890c49 100644 --- a/src/Dictionaries/PolygonDictionary.h +++ b/src/Dictionaries/PolygonDictionary.h @@ -86,7 +86,7 @@ public: bool isInjective(const std::string & attribute_name) const override { return dict_struct.getAttribute(attribute_name).injective; } - DictionaryKeyType getKeyType() const override { return DictionaryKeyType::complex; } + DictionaryKeyType getKeyType() const override { return DictionaryKeyType::Complex; } ColumnPtr getColumn( const std::string& attribute_name, diff --git a/src/Dictionaries/RangeDictionarySource.h b/src/Dictionaries/RangeDictionarySource.h index 252ab97ac74..dcc5b1ea3b8 100644 --- a/src/Dictionaries/RangeDictionarySource.h +++ b/src/Dictionaries/RangeDictionarySource.h @@ -14,18 +14,12 @@ namespace DB { -enum class RangeDictionaryType -{ - simple, - complex -}; - -template +template class RangeDictionarySourceData { public: - using KeyType = std::conditional_t; + using KeyType = std::conditional_t; RangeDictionarySourceData( std::shared_ptr dictionary, @@ -58,8 +52,8 @@ private: }; -template -RangeDictionarySourceData::RangeDictionarySourceData( +template +RangeDictionarySourceData::RangeDictionarySourceData( std::shared_ptr dictionary_, const Names & column_names_, PaddedPODArray && keys, @@ -73,8 +67,8 @@ RangeDictionarySourceData::RangeDictionarySour { } -template -Block RangeDictionarySourceData::getBlock(size_t start, size_t length) const +template +Block RangeDictionarySourceData::getBlock(size_t start, size_t length) const { PaddedPODArray block_keys; PaddedPODArray block_start_dates; @@ -93,8 +87,8 @@ Block RangeDictionarySourceData::getBlock(size return fillBlock(block_keys, block_start_dates, block_end_dates, start, start + length); } -template -PaddedPODArray RangeDictionarySourceData::makeDateKeys( +template +PaddedPODArray RangeDictionarySourceData::makeDateKeys( const PaddedPODArray & block_start_dates, const PaddedPODArray & block_end_dates) const { @@ -112,24 +106,14 @@ PaddedPODArray RangeDictionarySourceData -Block RangeDictionarySourceData::fillBlock( +template +Block RangeDictionarySourceData::fillBlock( const PaddedPODArray & keys_to_fill, const PaddedPODArray & block_start_dates, const PaddedPODArray & block_end_dates, size_t start, size_t end) const { - std::cerr << "RangeDictionarySourceData::fillBlock keys_to_fill " << keys_to_fill.size() << std::endl; - - if constexpr (range_dictionary_type == RangeDictionaryType::simple) - { - for (auto & key : keys_to_fill) - { - std::cerr << key << std::endl; - } - } - ColumnsWithTypeAndName columns; const DictionaryStructure & dictionary_structure = dictionary->getStructure(); @@ -137,7 +121,7 @@ Block RangeDictionarySourceData::fillBlock( Columns keys_columns; Strings keys_names = dictionary_structure.getKeysNames(); - if constexpr (range_dictionary_type == RangeDictionaryType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) { keys_columns = {getColumnFromPODArray(keys_to_fill)}; keys_types = {std::make_shared()}; @@ -154,9 +138,6 @@ Block RangeDictionarySourceData::fillBlock( size_t keys_size = keys_names.size(); - std::cerr << "Keys size " << keys_size << " key columns size " << keys_columns.size(); - std::cerr << " keys types size " << keys_types.size() << std::endl; - assert(keys_columns.size() == keys_size); assert(keys_types.size() == keys_size); @@ -204,51 +185,33 @@ Block RangeDictionarySourceData::fillBlock( columns.emplace_back(std::move(column), attribute.type, attribute.name); } - auto result = Block(columns); - - Field value; - std::cerr << "RangeDictionarySourceData::fillBlock result" << std::endl; - for (auto & block_column : result) - { - std::cerr << "Column name " << block_column.name << " type " << block_column.type->getName() << std::endl; - - auto & column = block_column.column; - size_t column_size = column->size(); - - for (size_t i = 0; i < column_size; ++i) - { - column->get(i, value); - std::cerr << "Index " << i << " value " << value.dump() << std::endl; - } - } - return Block(columns); } -template +template class RangeDictionarySource : public DictionarySourceBase { public: - RangeDictionarySource(RangeDictionarySourceData data_, size_t max_block_size); + RangeDictionarySource(RangeDictionarySourceData data_, size_t max_block_size); String getName() const override { return "RangeDictionarySource"; } protected: Block getBlock(size_t start, size_t length) const override; - RangeDictionarySourceData data; + RangeDictionarySourceData data; }; -template -RangeDictionarySource::RangeDictionarySource(RangeDictionarySourceData data_, size_t max_block_size) +template +RangeDictionarySource::RangeDictionarySource(RangeDictionarySourceData data_, size_t max_block_size) : DictionarySourceBase(data_.getBlock(0, 0), data_.getNumRows(), max_block_size) , data(std::move(data_)) { } -template -Block RangeDictionarySource::getBlock(size_t start, size_t length) const +template +Block RangeDictionarySource::getBlock(size_t start, size_t length) const { return data.getBlock(start, length); } diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 50935163a96..ea0af493bdf 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -95,6 +95,13 @@ ColumnPtr RangeHashedDictionary::getColumn( const DataTypes & key_types, const ColumnPtr & default_values_column) const { + if (dictionary_key_type == DictionaryKeyType::Complex) + { + auto key_types_copy = key_types; + key_types_copy.pop_back(); + dict_struct.validateKeyTypes(key_types_copy); + } + ColumnPtr result; const auto & dictionary_attribute = dict_struct.getAttribute(attribute_name, result_type); @@ -206,9 +213,15 @@ ColumnPtr RangeHashedDictionary::getColumn( template ColumnUInt8::Ptr RangeHashedDictionary::hasKeys(const Columns & key_columns, const DataTypes & key_types) const { + if (dictionary_key_type == DictionaryKeyType::Complex) + { + auto key_types_copy = key_types; + key_types_copy.pop_back(); + dict_struct.validateKeyTypes(key_types_copy); + } auto range_column_storage_type = std::make_shared(); auto range_storage_column = key_columns.back(); - ColumnWithTypeAndName column_to_cast = {range_storage_column->convertToFullColumnIfConst(), key_types[1], ""}; + ColumnWithTypeAndName column_to_cast = {range_storage_column->convertToFullColumnIfConst(), key_types.back(), ""}; auto range_column_updated = castColumnAccurate(column_to_cast, range_column_storage_type); PaddedPODArray range_backup_storage; const PaddedPODArray & dates = getColumnVectorData(this, range_column_updated, range_backup_storage); @@ -383,7 +396,7 @@ void RangeHashedDictionary::calculateBytesAllocated() callOnDictionaryAttributeType(attribute.type, type_call); } - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) bytes_allocated += complex_key_arena.size(); } @@ -607,10 +620,9 @@ Pipe RangeHashedDictionary::readImpl(const Names & column_n PaddedPODArray end_dates; getKeysAndDates(keys, start_dates, end_dates); - static constexpr RangeDictionaryType range_dictionary_type = (dictionary_key_type == DictionaryKeyType::simple) ? RangeDictionaryType::simple : RangeDictionaryType::complex; - using RangeDictionarySourceType = RangeDictionarySource; + using RangeDictionarySourceType = RangeDictionarySource; - auto source_data = RangeDictionarySourceData( + auto source_data = RangeDictionarySourceData( shared_from_this(), column_names, std::move(keys), @@ -690,7 +702,7 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory) const auto dict_id = StorageID::fromDictionaryConfig(config, config_prefix); const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); - return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); + return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); }; factory.registerLayout("range_hashed", create_layout_simple, false); @@ -713,7 +725,7 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory) const auto dict_id = StorageID::fromDictionaryConfig(config, config_prefix); const DictionaryLifetime dict_lifetime{config, config_prefix + ".lifetime"}; const bool require_nonempty = config.getBool(config_prefix + ".require_nonempty", false); - return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); + return std::make_unique>(dict_id, dict_struct, std::move(source_ptr), dict_lifetime, require_nonempty); }; factory.registerLayout("complex_key_range_hashed", create_layout_complex, true); } diff --git a/src/Dictionaries/RangeHashedDictionary.h b/src/Dictionaries/RangeHashedDictionary.h index f9b09189265..4cdab66a0e4 100644 --- a/src/Dictionaries/RangeHashedDictionary.h +++ b/src/Dictionaries/RangeHashedDictionary.h @@ -32,8 +32,7 @@ template class RangeHashedDictionary final : public IDictionary { public: - using KeyType = std::conditional_t; - static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by hashed dictionary"); + using KeyType = std::conditional_t; RangeHashedDictionary( const StorageID & dict_id_, @@ -78,7 +77,9 @@ public: return dict_struct.getAttribute(attribute_name).injective; } - DictionaryKeyType getKeyType() const override { return DictionaryKeyType::range; } + DictionaryKeyType getKeyType() const override { return dictionary_key_type; } + + DictionarySpecialKeyType getSpecialKeyType() const override { return DictionarySpecialKeyType::Range;} ColumnPtr getColumn( const std::string& attribute_name, @@ -104,7 +105,7 @@ private: template using CollectionType = std::conditional_t< - dictionary_key_type == DictionaryKeyType::simple, + dictionary_key_type == DictionaryKeyType::Simple, HashMap>, HashMapWithSavedHash, DefaultHash>>; diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index bdb640c90be..e584b523376 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -823,8 +823,8 @@ template class SSDCacheDictionaryStorage final : public ICacheDictionaryStorage { public: - using SSDCacheKeyType = std::conditional_t; - using KeyType = std::conditional_t; + using SSDCacheKeyType = std::conditional_t; + using KeyType = std::conditional_t; explicit SSDCacheDictionaryStorage(const SSDCacheDictionaryStorageConfiguration & configuration_) : configuration(configuration_) @@ -838,19 +838,19 @@ public: String getName() const override { - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) return "SSDCache"; else return "SSDComplexKeyCache"; } - bool supportsSimpleKeys() const override { return dictionary_key_type == DictionaryKeyType::simple; } + bool supportsSimpleKeys() const override { return dictionary_key_type == DictionaryKeyType::Simple; } SimpleKeysStorageFetchResult fetchColumnsForKeys( const PaddedPODArray & keys, const DictionaryStorageFetchRequest & fetch_request) override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) return fetchColumnsForKeysImpl(keys, fetch_request); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertColumnsForKeys is not supported for complex key storage"); @@ -858,7 +858,7 @@ public: void insertColumnsForKeys(const PaddedPODArray & keys, Columns columns) override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) insertColumnsForKeysImpl(keys, columns); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertColumnsForKeys is not supported for complex key storage"); @@ -866,7 +866,7 @@ public: void insertDefaultKeys(const PaddedPODArray & keys) override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) insertDefaultKeysImpl(keys); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertDefaultKeysImpl is not supported for complex key storage"); @@ -874,19 +874,19 @@ public: PaddedPODArray getCachedSimpleKeys() const override { - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) return getCachedKeysImpl(); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getCachedSimpleKeys is not supported for complex key storage"); } - bool supportsComplexKeys() const override { return dictionary_key_type == DictionaryKeyType::complex; } + bool supportsComplexKeys() const override { return dictionary_key_type == DictionaryKeyType::Complex; } ComplexKeysStorageFetchResult fetchColumnsForKeys( const PaddedPODArray & keys, const DictionaryStorageFetchRequest & fetch_request) override { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) return fetchColumnsForKeysImpl(keys, fetch_request); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method fetchColumnsForKeys is not supported for simple key storage"); @@ -894,7 +894,7 @@ public: void insertColumnsForKeys(const PaddedPODArray & keys, Columns columns) override { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) insertColumnsForKeysImpl(keys, columns); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertColumnsForKeys is not supported for simple key storage"); @@ -902,7 +902,7 @@ public: void insertDefaultKeys(const PaddedPODArray & keys) override { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) insertDefaultKeysImpl(keys); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method insertDefaultKeysImpl is not supported for simple key storage"); @@ -910,7 +910,7 @@ public: PaddedPODArray getCachedComplexKeys() const override { - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) return getCachedKeysImpl(); else throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Method getCachedSimpleKeys is not supported for simple key storage"); @@ -1134,7 +1134,7 @@ private: Cell cell; setCellDeadline(cell, now); - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) { /// Copy complex key into arena and put in cache size_t key_size = key.size; @@ -1166,7 +1166,7 @@ private: cell.state = Cell::default_value; - if constexpr (dictionary_key_type == DictionaryKeyType::complex) + if constexpr (dictionary_key_type == DictionaryKeyType::Complex) { /// Copy complex key into arena and put in cache size_t key_size = key.size; @@ -1382,7 +1382,7 @@ private: using ComplexKeyHashMap = HashMapWithSavedHash; using CacheMap = std::conditional_t< - dictionary_key_type == DictionaryKeyType::simple, + dictionary_key_type == DictionaryKeyType::Simple, SimpleKeyHashMap, ComplexKeyHashMap>; diff --git a/src/Dictionaries/registerCacheDictionaries.cpp b/src/Dictionaries/registerCacheDictionaries.cpp index d039c5b6630..64c1c55e0ba 100644 --- a/src/Dictionaries/registerCacheDictionaries.cpp +++ b/src/Dictionaries/registerCacheDictionaries.cpp @@ -157,24 +157,23 @@ DictionaryPtr createCacheDictionaryLayout( ContextPtr context [[maybe_unused]], bool created_from_ddl [[maybe_unused]]) { - static_assert(dictionary_key_type != DictionaryKeyType::range, "Range key type is not supported by CacheDictionary"); - String layout_type; - if constexpr (dictionary_key_type == DictionaryKeyType::simple && !ssd) + + if constexpr (dictionary_key_type == DictionaryKeyType::Simple && !ssd) layout_type = "cache"; - else if constexpr (dictionary_key_type == DictionaryKeyType::simple && ssd) + else if constexpr (dictionary_key_type == DictionaryKeyType::Simple && ssd) layout_type = "ssd_cache"; - else if constexpr (dictionary_key_type == DictionaryKeyType::complex && !ssd) + else if constexpr (dictionary_key_type == DictionaryKeyType::Complex && !ssd) layout_type = "complex_key_cache"; - else if constexpr (dictionary_key_type == DictionaryKeyType::complex && ssd) + else if constexpr (dictionary_key_type == DictionaryKeyType::Complex && ssd) layout_type = "complex_key_ssd_cache"; - if constexpr (dictionary_key_type == DictionaryKeyType::simple) + if constexpr (dictionary_key_type == DictionaryKeyType::Simple) { if (dict_struct.key) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "{}: dictionary of layout '{}' 'key' is not supported", full_name, layout_type); } - else if constexpr (dictionary_key_type == DictionaryKeyType::complex) + else if constexpr (dictionary_key_type == DictionaryKeyType::Complex) { if (dict_struct.id) throw Exception(ErrorCodes::UNSUPPORTED_METHOD, "{}: dictionary of layout '{}' 'id' is not supported", full_name, layout_type); @@ -243,7 +242,7 @@ void registerDictionaryCache(DictionaryFactory & factory) ContextPtr context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); }; factory.registerLayout("cache", create_simple_cache_layout, false); @@ -256,7 +255,7 @@ void registerDictionaryCache(DictionaryFactory & factory) ContextPtr context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); }; factory.registerLayout("complex_key_cache", create_complex_key_cache_layout, true); @@ -271,7 +270,7 @@ void registerDictionaryCache(DictionaryFactory & factory) ContextPtr context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); }; factory.registerLayout("ssd_cache", create_simple_ssd_cache_layout, false); @@ -283,7 +282,7 @@ void registerDictionaryCache(DictionaryFactory & factory) DictionarySourcePtr source_ptr, ContextPtr context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); }; factory.registerLayout("complex_key_ssd_cache", create_complex_key_ssd_cache_layout, true); diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 02af528723c..8d22b233ffe 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -182,15 +182,16 @@ public: auto dictionary = helper.getDictionary(arguments[0].column); auto dictionary_key_type = dictionary->getKeyType(); + auto dictionary_special_key_type = dictionary->getSpecialKeyType(); - const ColumnWithTypeAndName & key_column_with_type = arguments[1]; + const auto & key_column_with_type = arguments[1]; auto key_column = key_column_with_type.column; auto key_column_type = key_column_with_type.type; - ColumnPtr range_col = nullptr; - DataTypePtr range_col_type = nullptr; + ColumnPtr range_col; + DataTypePtr range_col_type; - if (dictionary_key_type == DictionaryKeyType::range) + if (dictionary_special_key_type == DictionarySpecialKeyType::Range) { if (arguments.size() != 3) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, @@ -207,7 +208,10 @@ public: getName()); } - if (dictionary_key_type == DictionaryKeyType::simple) + Columns key_columns; + DataTypes key_types; + + if (dictionary_key_type == DictionaryKeyType::Simple) { if (!WhichDataType(key_column_type).isUInt64()) throw Exception( @@ -216,16 +220,23 @@ public: getName(), key_column_with_type.type->getName()); - return dictionary->hasKeys({key_column}, {std::make_shared()}); + key_columns = {key_column}; + key_types = {std::make_shared()}; } - else if (dictionary_key_type == DictionaryKeyType::complex) + else if (dictionary_key_type == DictionaryKeyType::Complex) { /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys. key_column = key_column->convertToFullColumnIfConst(); - size_t keys_size = dictionary->getStructure().getKeysSize(); - if (!isTuple(key_column_type)) + if (isTuple(key_column_type)) { + key_columns = assert_cast(*key_column).getColumnsCopy(); + key_types = assert_cast(*key_column_type).getElements(); + } + else + { + size_t keys_size = dictionary->getStructure().getKeysSize(); + if (keys_size > 1) { throw Exception( @@ -237,41 +248,24 @@ public: } else { - Columns tuple_columns = {std::move(key_column)}; - key_column = ColumnTuple::create(tuple_columns); - - DataTypes tuple_types = {key_column_type}; - key_column_type = std::make_shared(tuple_types); + key_columns = {key_column}; + key_types = {key_column_type}; } } - - const auto & key_columns = assert_cast(*key_column).getColumnsCopy(); - const auto & key_types = assert_cast(*key_column_type).getElements(); - - return dictionary->hasKeys(key_columns, key_types); } - else + + if (dictionary_special_key_type == DictionarySpecialKeyType::Range) { - /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys. - ColumnPtr key_column = key_column_with_type.column->convertToFullColumnIfConst(); - DataTypePtr key_column_type = key_column_with_type.type; - - Columns key_columns; - DataTypes key_types; - - if (isTuple(key_column_type)) - { - key_columns = assert_cast(*key_column).getColumnsCopy(); - key_types = assert_cast(*key_column_type).getElements(); - } - else - { - key_columns = {key_column, range_col}; - key_types = {std::make_shared(), range_col_type}; - } - - return dictionary->hasKeys({key_column, range_col}, {std::make_shared(), range_col_type}); + key_columns.emplace_back(range_col); + key_types.emplace_back(range_col_type); } + + std::cerr << "FunctionDictHas::executeImpl" << std::endl; + + for (auto & key_type : key_types) + std::cerr << "Key type " << key_type->getName() << std::endl; + + return dictionary->hasKeys(key_columns, key_types); } private: @@ -369,13 +363,14 @@ public: auto dictionary = helper.getDictionary(dictionary_name); auto dictionary_key_type = dictionary->getKeyType(); + auto dictionary_special_key_type = dictionary->getSpecialKeyType(); size_t current_arguments_index = 3; ColumnPtr range_col = nullptr; DataTypePtr range_col_type = nullptr; - if (dictionary_key_type == DictionaryKeyType::range) + if (dictionary_special_key_type == DictionarySpecialKeyType::Range) { if (current_arguments_index >= arguments.size()) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, @@ -437,12 +432,13 @@ public: default_cols.emplace_back(nullptr); } - ColumnPtr result; + const auto & key_col_with_type = arguments[2]; + auto key_column = key_col_with_type.column; - const ColumnWithTypeAndName & key_col_with_type = arguments[2]; - const auto key_column = key_col_with_type.column; + Columns key_columns; + DataTypes key_types; - if (dictionary_key_type == DictionaryKeyType::simple) + if (dictionary_key_type == DictionaryKeyType::Simple) { if (!WhichDataType(key_col_with_type.type).isUInt64()) throw Exception( @@ -451,24 +447,24 @@ public: getName(), key_col_with_type.type->getName()); - result = executeDictionaryRequest( - dictionary, - attribute_names, - {key_column}, - {std::make_shared()}, - result_type, - default_cols); + key_columns = {key_column}; + key_types = {std::make_shared()}; } - else if (dictionary_key_type == DictionaryKeyType::complex) + else if (dictionary_key_type == DictionaryKeyType::Complex) { /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys. - ColumnPtr key_column = key_col_with_type.column->convertToFullColumnIfConst(); + key_column = key_column->convertToFullColumnIfConst(); DataTypePtr key_column_type = key_col_with_type.type; - size_t keys_size = dictionary->getStructure().getKeysSize(); - - if (!isTuple(key_column_type)) + if (isTuple(key_column_type)) { + key_columns = assert_cast(*key_column).getColumnsCopy(); + key_types = assert_cast(*key_column_type).getElements(); + } + else if (!isTuple(key_column_type)) + { + size_t keys_size = dictionary->getStructure().getKeysSize(); + if (keys_size > 1) { throw Exception( @@ -480,60 +476,19 @@ public: } else { - Columns tuple_columns = {std::move(key_column)}; - key_column = ColumnTuple::create(tuple_columns); - - DataTypes tuple_types = {key_column_type}; - key_column_type = std::make_shared(tuple_types); + key_columns = {std::move(key_column)}; + key_types = {std::move(key_column_type)}; } } - - const auto & key_columns = assert_cast(*key_column).getColumnsCopy(); - const auto & key_types = assert_cast(*key_column_type).getElements(); - - result = executeDictionaryRequest( - dictionary, - attribute_names, - key_columns, - key_types, - result_type, - default_cols); } - else if (dictionary_key_type == DictionaryKeyType::range) + + if (dictionary_special_key_type == DictionarySpecialKeyType::Range) { - /// Functions in external dictionaries_loader only support full-value (not constant) columns with keys. - ColumnPtr key_column = key_col_with_type.column->convertToFullColumnIfConst(); - DataTypePtr key_column_type = key_col_with_type.type; - - Columns key_columns; - DataTypes key_types; - - if (isTuple(key_column_type)) - { - key_columns = assert_cast(*key_column).getColumnsCopy(); - key_types = assert_cast(*key_column_type).getElements(); - } - else - { - key_columns = {key_column}; - key_types = {std::make_shared()}; - } - key_columns.emplace_back(range_col); key_types.emplace_back(range_col_type); - - result = executeDictionaryRequest( - dictionary, - attribute_names, - key_columns, - key_types, - result_type, - default_cols); } - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown dictionary identifier type"); - return result; + return executeDictionaryRequest(dictionary, attribute_names, key_columns, key_types, result_type, default_cols); } private: From baaaf996c2317d9c170fba5bfa7ed2d5d3c6ae81 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Tue, 17 Aug 2021 21:15:44 +0300 Subject: [PATCH 143/220] Update getSourceFromFromASTInsertQuery.cpp --- src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp index daf07a208bd..9e64bd954fa 100644 --- a/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromFromASTInsertQuery.cpp @@ -64,7 +64,8 @@ Pipe getSourceFromFromASTInsertQuery( const auto in_file = in_file_node.value.safeGet(); input_buffer = wrapReadBufferWithCompressionMethod(std::make_unique(in_file), chooseCompressionMethod(in_file, "")); - } else + } + else { ConcatReadBuffer::ReadBuffers buffers; if (ast_insert_query->data) From 0821338f14ab209e3561085c200095b419cc882e Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:50:01 +0300 Subject: [PATCH 144/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index 0642b8cbad3..db3ccd91d98 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -47,7 +47,7 @@ - `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в данных узла ZooKeeper. - `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в детях узла ZooKeeper. - `stat_dataLength` ([Int32](../../sql-reference/data-types/int-uint.md)) — длина поля данных узла ZooKeeper. -- `stat_numChildren` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество детей узла ZooKeeper. +- `stat_numChildren` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество дочерних узлов ZooKeeper. - `children` ([Array(String)](../../sql-reference/data-types/array.md)) — список дочерних узлов ZooKeeper (для ответов на запрос `LIST`). **Пример** From 28fd94d016f3315f1be0c39643c60d3cb75008e6 Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:50:15 +0300 Subject: [PATCH 145/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index db3ccd91d98..2683c10b80e 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -45,7 +45,7 @@ - `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — `zxid` изменения, которое последним модифицировало узел ZooKeeper. - `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции изменения, которое последним модифицировало детей узла ZooKeeper. - `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в данных узла ZooKeeper. -- `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в детях узла ZooKeeper. +- `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в дочерних узлах ZooKeeper. - `stat_dataLength` ([Int32](../../sql-reference/data-types/int-uint.md)) — длина поля данных узла ZooKeeper. - `stat_numChildren` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество дочерних узлов ZooKeeper. - `children` ([Array(String)](../../sql-reference/data-types/array.md)) — список дочерних узлов ZooKeeper (для ответов на запрос `LIST`). From 5f20c2b1f5062ea4570782f747443905c496fd54 Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:50:44 +0300 Subject: [PATCH 146/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index 2683c10b80e..b2899b51870 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -43,7 +43,7 @@ - `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как sequential. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — `zxid` изменения, в результате которого был создан узел ZooKeeper. - `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — `zxid` изменения, которое последним модифицировало узел ZooKeeper. -- `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции изменения, которое последним модифицировало детей узла ZooKeeper. +- `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала дочерние узлы ZooKeeper. - `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в данных узла ZooKeeper. - `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в дочерних узлах ZooKeeper. - `stat_dataLength` ([Int32](../../sql-reference/data-types/int-uint.md)) — длина поля данных узла ZooKeeper. From afbce1b90a53242f1ead0a6dca438765634321dd Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:51:05 +0300 Subject: [PATCH 147/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index b2899b51870..3c2799779da 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -126,7 +126,7 @@ stat_numChildren: 7 children: ['query-0000000006','query-0000000005','query-0000000004','query-0000000003','query-0000000002','query-0000000001','query-0000000000'] ``` -**Смотрите также** +**См. также** - [ZooKeeper](../../operations/tips.md#zookeeper) - [Руководство по ZooKeeper](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html) From e4229e0eeb65b15f4a1cfc1bb3ed15b7fff1db15 Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:51:23 +0300 Subject: [PATCH 148/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index 3c2799779da..ada5af4735f 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -42,7 +42,7 @@ - `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — статус события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. - `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как sequential. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — `zxid` изменения, в результате которого был создан узел ZooKeeper. -- `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — `zxid` изменения, которое последним модифицировало узел ZooKeeper. +- `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала узел ZooKeeper. - `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала дочерние узлы ZooKeeper. - `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в данных узла ZooKeeper. - `stat_cversion` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в дочерних узлах ZooKeeper. From 63ee49ee7d7e36eca100dac99593d94608559458 Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:51:47 +0300 Subject: [PATCH 149/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index ada5af4735f..ee7f133267d 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -41,7 +41,7 @@ - `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — тип события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. - `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — статус события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. - `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как sequential. -- `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — `zxid` изменения, в результате которого был создан узел ZooKeeper. +- `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, в результате которой был создан узел ZooKeeper. - `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала узел ZooKeeper. - `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала дочерние узлы ZooKeeper. - `stat_version` ([Int32](../../sql-reference/data-types/int-uint.md)) — количество изменений в данных узла ZooKeeper. From 496fdb2a97fc4380d39a6ca58de4b99295ece608 Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:52:47 +0300 Subject: [PATCH 150/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index ee7f133267d..2148e08d33b 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -17,7 +17,7 @@ - `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — дата и время завершения выполнения запроса. - `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого был сделан запрос. - `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт, с которого был сделан запрос. -- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор сессии, который сервер ZooKeeper устанавливает для каждого соединения. +- `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор сессии, который сервер ZooKeeper создает для каждого соединения. - `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — идентификатор запроса внутри сессии. Обычно это последовательный номер запроса, одинаковый у строки запроса и у парной строки `response`/`finalize`. - `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — установлен ли запрос [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches). - `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — тип запроса или ответа на запрос. From 3ef0e600ce7c548bea6be6d03e6a43cc5b5978bd Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:54:06 +0300 Subject: [PATCH 151/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index 2148e08d33b..afa7642a3b6 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -40,7 +40,7 @@ - `NULL` — выполнен запрос. - `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — тип события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. - `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — статус события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. -- `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как sequential. +- `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как `sequential`. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, в результате которой был создан узел ZooKeeper. - `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала узел ZooKeeper. - `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала дочерние узлы ZooKeeper. From c1860b2342e7e2752f24edc4e6926ae0ec3f440b Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:54:20 +0300 Subject: [PATCH 152/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index afa7642a3b6..f1290851a42 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -39,7 +39,7 @@ - `ZSESSIONEXPIRED` — истекло время сессии. - `NULL` — выполнен запрос. - `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — тип события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. -- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — статус события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. +- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — статус события `watch` (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. - `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как `sequential`. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, в результате которой был создан узел ZooKeeper. - `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, которая последней модифицировала узел ZooKeeper. From 7b8c723f1579c020a2cba1ed5fd1d029eec6248d Mon Sep 17 00:00:00 2001 From: Dmitriy <72220289+sevirov@users.noreply.github.com> Date: Tue, 17 Aug 2021 22:54:29 +0300 Subject: [PATCH 153/220] Update docs/ru/operations/system-tables/zookeeper_log.md Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/ru/operations/system-tables/zookeeper_log.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index f1290851a42..a78a5089bdf 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -38,7 +38,7 @@ - `ZOPERATIONTIMEOUT` — истекло время ожидания выполнения запроса. - `ZSESSIONEXPIRED` — истекло время сессии. - `NULL` — выполнен запрос. -- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — тип события watch (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. +- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — тип события `watch` (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. - `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — статус события `watch` (для ответов на запрос при `op_num` = `Watch`), для остальных ответов: `NULL`. - `path_created` ([String](../../sql-reference/data-types/string.md)) — путь к созданному узлу ZooKeeper (для ответов на запрос `CREATE`). Может отличаться от `path`, если узел создается как `sequential`. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции, в результате которой был создан узел ZooKeeper. From 495c359889f22adb6a2e62b0bd88c964f12c1348 Mon Sep 17 00:00:00 2001 From: Sergei Semin Date: Tue, 17 Aug 2021 22:58:29 +0300 Subject: [PATCH 154/220] Revert "add -Wno-reserved-identifier in necessary places" This reverts commit 84660f36de124ad5a6480a945ae8968f8381c3a3. --- utils/corrector_utf8/CMakeLists.txt | 1 - utils/iotest/CMakeLists.txt | 2 -- utils/zookeeper-cli/CMakeLists.txt | 1 - 3 files changed, 4 deletions(-) diff --git a/utils/corrector_utf8/CMakeLists.txt b/utils/corrector_utf8/CMakeLists.txt index a426815bf99..4784fd43e2d 100644 --- a/utils/corrector_utf8/CMakeLists.txt +++ b/utils/corrector_utf8/CMakeLists.txt @@ -1,3 +1,2 @@ add_executable(corrector_utf8 corrector_utf8.cpp) target_link_libraries(corrector_utf8 PRIVATE clickhouse_common_io) -target_no_warning(corrector_utf8 reserved-identifier) diff --git a/utils/iotest/CMakeLists.txt b/utils/iotest/CMakeLists.txt index 66e2b982104..8f141b178f0 100644 --- a/utils/iotest/CMakeLists.txt +++ b/utils/iotest/CMakeLists.txt @@ -4,8 +4,6 @@ target_link_libraries (iotest PRIVATE clickhouse_common_io) add_executable (iotest_nonblock iotest_nonblock.cpp ${SRCS}) target_link_libraries (iotest_nonblock PRIVATE clickhouse_common_io) -target_no_warning(iotest_nonblock reserved-identifier) add_executable (iotest_aio iotest_aio.cpp ${SRCS}) target_link_libraries (iotest_aio PRIVATE clickhouse_common_io) -target_no_warning(iotest_aio reserved-identifier) diff --git a/utils/zookeeper-cli/CMakeLists.txt b/utils/zookeeper-cli/CMakeLists.txt index 90794dcceb5..2199a1b38ff 100644 --- a/utils/zookeeper-cli/CMakeLists.txt +++ b/utils/zookeeper-cli/CMakeLists.txt @@ -1,3 +1,2 @@ add_executable(clickhouse-zookeeper-cli zookeeper-cli.cpp) target_link_libraries(clickhouse-zookeeper-cli PRIVATE clickhouse_common_zookeeper) -target_no_warning(clickhouse-zookeeper-cli reserved-identifier) From 693b8271dc3ff610635f31530f2a99b452806d23 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Tue, 17 Aug 2021 23:05:21 +0300 Subject: [PATCH 155/220] Fix typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Выделил watch и sequental как ключевые слова. --- docs/en/operations/system-tables/zookeeper_log.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index 25d2d186724..5585b1a6dcd 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -38,9 +38,9 @@ Columns with request response parameters: - `ZOPERATIONTIMEOUT` — The request execution timeout has expired. - `ZSESSIONEXPIRED` — The session has expired. - `NULL` — The request is completed. -- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the watch event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. -- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the watch event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. -- `path_created` ([String](../../sql-reference/data-types/string.md)) — The path to the created ZooKeeper node (for responses to the `CREATE` request), may differ from the `path` if the node is created as a sequential. +- `watch_type` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The type of the `watch` event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. +- `watch_state` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — The status of the `watch` event (for responses with `op_num` = `Watch`), for the remaining responses: `NULL`. +- `path_created` ([String](../../sql-reference/data-types/string.md)) — The path to the created ZooKeeper node (for responses to the `CREATE` request), may differ from the `path` if the node is created as a `sequential`. - `stat_czxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that caused this ZooKeeper node to be created. - `stat_mzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The `zxid` of the change that last modified this ZooKeeper node. - `stat_pzxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — The transaction ID of the change that last modified childern of this ZooKeeper node. From da3f5612ae5677bdf0523ac7ba06ed5c1309164a Mon Sep 17 00:00:00 2001 From: Sergei Semin Date: Tue, 17 Aug 2021 23:08:17 +0300 Subject: [PATCH 156/220] rename _Bits -> _bits --- base/common/wide_integer_impl.h | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/base/common/wide_integer_impl.h b/base/common/wide_integer_impl.h index d2ef8b22d65..27efe18eedd 100644 --- a/base/common/wide_integer_impl.h +++ b/base/common/wide_integer_impl.h @@ -152,7 +152,7 @@ namespace wide template struct integer::_impl { - static constexpr size_t _Bits = Bits; + static constexpr size_t _bits = Bits; static constexpr const unsigned byte_count = Bits / 8; static constexpr const unsigned item_count = byte_count / sizeof(base_type); static constexpr const unsigned base_bits = sizeof(base_type) * 8; @@ -614,8 +614,8 @@ public: else { static_assert(IsWideInteger::value); - return std::common_type_t, integer>::_impl::operator_plus( - integer(lhs), rhs); + return std::common_type_t, integer>::_impl::operator_plus( + integer(lhs), rhs); } } @@ -632,8 +632,8 @@ public: else { static_assert(IsWideInteger::value); - return std::common_type_t, integer>::_impl::operator_minus( - integer(lhs), rhs); + return std::common_type_t, integer>::_impl::operator_minus( + integer(lhs), rhs); } } @@ -857,7 +857,7 @@ public: else { static_assert(IsWideInteger::value); - return std::common_type_t, integer>::operator_slash(T(lhs), rhs); + return std::common_type_t, integer>::operator_slash(T(lhs), rhs); } } @@ -877,7 +877,7 @@ public: else { static_assert(IsWideInteger::value); - return std::common_type_t, integer>::operator_percent(T(lhs), rhs); + return std::common_type_t, integer>::operator_percent(T(lhs), rhs); } } From 6c82ac2024c9fc646022e089d9cef2277c704fda Mon Sep 17 00:00:00 2001 From: Maksim Kita Date: Tue, 17 Aug 2021 23:59:08 +0300 Subject: [PATCH 157/220] Fixed tests --- src/Functions/FunctionsExternalDictionaries.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Functions/FunctionsExternalDictionaries.h b/src/Functions/FunctionsExternalDictionaries.h index 8d22b233ffe..5f94a1e1f4b 100644 --- a/src/Functions/FunctionsExternalDictionaries.h +++ b/src/Functions/FunctionsExternalDictionaries.h @@ -42,7 +42,6 @@ namespace ErrorCodes extern const int UNSUPPORTED_METHOD; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_COLUMN; - extern const int BAD_ARGUMENTS; extern const int TYPE_MISMATCH; } @@ -260,11 +259,6 @@ public: key_types.emplace_back(range_col_type); } - std::cerr << "FunctionDictHas::executeImpl" << std::endl; - - for (auto & key_type : key_types) - std::cerr << "Key type " << key_type->getName() << std::endl; - return dictionary->hasKeys(key_columns, key_types); } From e33a2bf7bc969b076674a6700f3d66fc6a651711 Mon Sep 17 00:00:00 2001 From: Kevin Michel Date: Wed, 21 Jul 2021 16:01:42 +0200 Subject: [PATCH 158/220] Add ReplicatedAccessStorage This stores Access Entities in ZooKeeper and replicates them across an entire cluster. This can be enabled by using the following configuration : /clickhouse/access/ --- .../settings.md | 4 + src/Access/AccessControlManager.cpp | 22 + src/Access/AccessControlManager.h | 4 + src/Access/ReplicatedAccessStorage.cpp | 618 ++++++++++++++++++ src/Access/ReplicatedAccessStorage.h | 87 +++ src/Access/ya.make | 1 + .../test_replicated_users/__init__.py | 0 .../test_replicated_users/configs/config.xml | 22 + .../integration/test_replicated_users/test.py | 73 +++ 9 files changed, 831 insertions(+) create mode 100644 src/Access/ReplicatedAccessStorage.cpp create mode 100644 src/Access/ReplicatedAccessStorage.h create mode 100644 tests/integration/test_replicated_users/__init__.py create mode 100644 tests/integration/test_replicated_users/configs/config.xml create mode 100644 tests/integration/test_replicated_users/test.py diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index aedd1c107c4..a02fa2418c9 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -1219,6 +1219,7 @@ Default value: `/var/lib/clickhouse/access/`. Section of the configuration file that contains settings: - Path to configuration file with predefined users. - Path to folder where users created by SQL commands are stored. +- ZooKeeper node path where users created by SQL commands are stored and replicated (experimental). If this section is specified, the path from [users_config](../../operations/server-configuration-parameters/settings.md#users-config) and [access_control_path](../../operations/server-configuration-parameters/settings.md#access_control_path) won't be used. @@ -1234,6 +1235,9 @@ The `user_directories` section can contain any number of items, the order of the /var/lib/clickhouse/access/ + + /clickhouse/access/ + ``` diff --git a/src/Access/AccessControlManager.cpp b/src/Access/AccessControlManager.cpp index 08f1cda2fce..95701bd42df 100644 --- a/src/Access/AccessControlManager.cpp +++ b/src/Access/AccessControlManager.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -225,6 +226,22 @@ void AccessControlManager::startPeriodicReloadingUsersConfigs() } } +void AccessControlManager::addReplicatedStorage( + const String & storage_name_, + const String & zookeeper_path_, + const zkutil::GetZooKeeper & get_zookeeper_function_) +{ + auto storages = getStoragesPtr(); + for (const auto & storage : *storages) + { + if (auto replicated_storage = typeid_cast>(storage)) + return; + } + auto new_storage = std::make_shared(storage_name_, zookeeper_path_, get_zookeeper_function_); + addStorage(new_storage); + LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName()); + new_storage->startup(); +} void AccessControlManager::addDiskStorage(const String & directory_, bool readonly_) { @@ -322,6 +339,11 @@ void AccessControlManager::addStoragesFromUserDirectoriesConfig( { addLDAPStorage(name, config, prefix); } + else if (type == ReplicatedAccessStorage::STORAGE_TYPE) + { + String zookeeper_path = config.getString(prefix + ".zookeeper_path"); + addReplicatedStorage(name, zookeeper_path, get_zookeeper_function); + } else throw Exception("Unknown storage type '" + type + "' at " + prefix + " in config", ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); } diff --git a/src/Access/AccessControlManager.h b/src/Access/AccessControlManager.h index e41aa80a257..15079ea61be 100644 --- a/src/Access/AccessControlManager.h +++ b/src/Access/AccessControlManager.h @@ -84,6 +84,10 @@ public: /// Adds LDAPAccessStorage which allows querying remote LDAP server for user info. void addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_); + void addReplicatedStorage(const String & storage_name, + const String & zookeeper_path, + const zkutil::GetZooKeeper & get_zookeeper_function); + /// Adds storages from config. void addStoragesFromUserDirectoriesConfig(const Poco::Util::AbstractConfiguration & config, const String & key, diff --git a/src/Access/ReplicatedAccessStorage.cpp b/src/Access/ReplicatedAccessStorage.cpp new file mode 100644 index 00000000000..f91b7c8fb06 --- /dev/null +++ b/src/Access/ReplicatedAccessStorage.cpp @@ -0,0 +1,618 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ +extern const int BAD_ARGUMENTS; +extern const int NO_ZOOKEEPER; +} + +static UUID parseUUID(const String & text) +{ + UUID uuid = UUIDHelpers::Nil; + auto buffer = ReadBufferFromMemory(text.data(), text.length()); + readUUIDText(uuid, buffer); + return uuid; +} + +ReplicatedAccessStorage::ReplicatedAccessStorage( + const String & storage_name_, + const String & zookeeper_path_, + zkutil::GetZooKeeper get_zookeeper_) + : IAccessStorage(storage_name_) + , zookeeper_path(zookeeper_path_) + , get_zookeeper(get_zookeeper_) +{ + if (zookeeper_path.empty()) + throw Exception("ZooKeeper path must be non-empty", ErrorCodes::BAD_ARGUMENTS); + + if (zookeeper_path.back() == '/') + zookeeper_path.resize(zookeeper_path.size() - 1); + + /// If zookeeper chroot prefix is used, path should start with '/', because chroot concatenates without it. + if (zookeeper_path.front() != '/') + zookeeper_path = "/" + zookeeper_path; +} + +ReplicatedAccessStorage::~ReplicatedAccessStorage() +{ + ReplicatedAccessStorage::shutdown(); +} + + +void ReplicatedAccessStorage::startup() +{ + initializeZookeeper(); + worker_thread = ThreadFromGlobalPool(&ReplicatedAccessStorage::runWorkerThread, this); +} + +void ReplicatedAccessStorage::shutdown() +{ + bool prev_stop_flag = stop_flag.exchange(true); + if (!prev_stop_flag) + { + /// Notify the worker thread to stop waiting for new queue items + refresh_queue.push(UUIDHelpers::Nil); + worker_thread.join(); + } +} + +template +static void retryOnZooKeeperUserError(size_t attempts, Func && function) +{ + while (attempts > 0) + { + try + { + function(); + return; + } + catch (zkutil::KeeperException & keeper_exception) + { + if (Coordination::isUserError(keeper_exception.code) && attempts > 1) + attempts -= 1; + else + throw; + } + } +} + +UUID ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists) +{ + const UUID id = generateRandomID(); + const EntityTypeInfo type_info = EntityTypeInfo::get(new_entity->getType()); + const String & name = new_entity->getName(); + LOG_DEBUG(getLogger(), "Inserting entity of type {} named {} with id {}", type_info.name, name, toString(id)); + + auto zookeeper = get_zookeeper(); + retryOnZooKeeperUserError(10, [&]{ insertZooKeeper(zookeeper, id, new_entity, replace_if_exists); }); + + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + std::lock_guard lock{mutex}; + refreshEntityNoLock(zookeeper, id, notifications); + return id; +} + + +void ReplicatedAccessStorage::insertZooKeeper( + const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists) +{ + const String & name = new_entity->getName(); + const EntityType type = new_entity->getType(); + const EntityTypeInfo type_info = EntityTypeInfo::get(type); + + const String entity_uuid = toString(id); + /// The entity data will be stored here, this ensures all entities have unique ids + const String entity_path = zookeeper_path + "/uuid/" + entity_uuid; + /// Then we create a znode with the entity name, inside the znode of each entity type + /// This ensure all entities of the same type have a unique name + const String name_path = zookeeper_path + "/" + type_info.unique_char + "/" + escapeForFileName(name); + + Coordination::Requests ops; + const String new_entity_definition = serializeAccessEntity(*new_entity); + ops.emplace_back(zkutil::makeCreateRequest(entity_path, new_entity_definition, zkutil::CreateMode::Persistent)); + /// The content of the "name" znode is the uuid of the entity owning that name + ops.emplace_back(zkutil::makeCreateRequest(name_path, entity_uuid, zkutil::CreateMode::Persistent)); + + Coordination::Responses responses; + const Coordination::Error res = zookeeper->tryMulti(ops, responses); + if (res == Coordination::Error::ZNODEEXISTS) + { + if (responses[0]->error == Coordination::Error::ZNODEEXISTS) + { + /// The UUID already exists, simply fail. + + /// To fail with a nice error message, we need info about what already exists. + /// This itself could fail if the conflicting uuid disappears in the meantime. + /// If that happens, then we'll just retry from the start. + String existing_entity_definition = zookeeper->get(entity_path); + + AccessEntityPtr existing_entity = deserializeAccessEntity(existing_entity_definition, entity_path); + EntityType existing_type = existing_entity->getType(); + String existing_name = existing_entity->getName(); + throwIDCollisionCannotInsert(id, type, name, existing_type, existing_name); + } + else if (replace_if_exists) + { + /// The name already exists for this type. + /// If asked to, we need to replace the existing entity. + + /// First get the uuid of the existing entity + /// This itself could fail if the conflicting name disappears in the meantime. + /// If that happens, then we'll just retry from the start. + Coordination::Stat name_stat; + String existing_entity_uuid = zookeeper->get(name_path, &name_stat); + + const String existing_entity_path = zookeeper_path + "/uuid/" + existing_entity_uuid; + Coordination::Requests replace_ops; + replace_ops.emplace_back(zkutil::makeRemoveRequest(existing_entity_path, -1)); + replace_ops.emplace_back(zkutil::makeCreateRequest(entity_path, new_entity_definition, zkutil::CreateMode::Persistent)); + replace_ops.emplace_back(zkutil::makeSetRequest(name_path, entity_uuid, name_stat.version)); + + /// If this fails, then we'll just retry from the start. + zookeeper->multi(replace_ops); + } + else + { + throwNameCollisionCannotInsert(type, name); + } + } + else + { + zkutil::KeeperMultiException::check(res, ops, responses); + } +} + +void ReplicatedAccessStorage::removeImpl(const UUID & id) +{ + LOG_DEBUG(getLogger(), "Removing entity {}", toString(id)); + + auto zookeeper = get_zookeeper(); + retryOnZooKeeperUserError(10, [&] { removeZooKeeper(zookeeper, id); }); + + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + std::lock_guard lock{mutex}; + removeEntityNoLock(id, notifications); +} + + +void ReplicatedAccessStorage::removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) +{ + const String entity_uuid = toString(id); + const String entity_path = zookeeper_path + "/uuid/" + entity_uuid; + + String entity_definition; + Coordination::Stat entity_stat; + const bool uuid_exists = zookeeper->tryGet(entity_path, entity_definition, &entity_stat); + if (!uuid_exists) + throwNotFound(id); + + const AccessEntityPtr entity = deserializeAccessEntity(entity_definition, entity_path); + const EntityTypeInfo type_info = EntityTypeInfo::get(entity->getType()); + const String & name = entity->getName(); + + const String entity_name_path = zookeeper_path + "/" + type_info.unique_char + "/" + escapeForFileName(name); + + Coordination::Requests ops; + ops.emplace_back(zkutil::makeRemoveRequest(entity_path, entity_stat.version)); + ops.emplace_back(zkutil::makeRemoveRequest(entity_name_path, -1)); + /// If this fails, then we'll just retry from the start. + zookeeper->multi(ops); +} + + +void ReplicatedAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func) +{ + LOG_DEBUG(getLogger(), "Updating entity {}", toString(id)); + + auto zookeeper = get_zookeeper(); + retryOnZooKeeperUserError(10, [&] { updateZooKeeper(zookeeper, id, update_func); }); + + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + std::lock_guard lock{mutex}; + refreshEntityNoLock(zookeeper, id, notifications); +} + + +void ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func) +{ + const String entity_uuid = toString(id); + const String entity_path = zookeeper_path + "/uuid/" + entity_uuid; + + String old_entity_definition; + Coordination::Stat stat; + const bool uuid_exists = zookeeper->tryGet(entity_path, old_entity_definition, &stat); + if (!uuid_exists) + throwNotFound(id); + + const AccessEntityPtr old_entity = deserializeAccessEntity(old_entity_definition, entity_path); + const AccessEntityPtr new_entity = update_func(old_entity); + + if (!new_entity->isTypeOf(old_entity->getType())) + throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType()); + + const EntityTypeInfo type_info = EntityTypeInfo::get(new_entity->getType()); + + Coordination::Requests ops; + const String new_entity_definition = serializeAccessEntity(*new_entity); + ops.emplace_back(zkutil::makeSetRequest(entity_path, new_entity_definition, stat.version)); + + const String & old_name = old_entity->getName(); + const String & new_name = new_entity->getName(); + if (new_name != old_name) + { + auto old_name_path = zookeeper_path + "/" + type_info.unique_char + "/" + escapeForFileName(old_name); + auto new_name_path = zookeeper_path + "/" + type_info.unique_char + "/" + escapeForFileName(new_name); + ops.emplace_back(zkutil::makeRemoveRequest(old_name_path, -1)); + ops.emplace_back(zkutil::makeCreateRequest(new_name_path, entity_uuid, zkutil::CreateMode::Persistent)); + } + + Coordination::Responses responses; + const Coordination::Error res = zookeeper->tryMulti(ops, responses); + if (res == Coordination::Error::ZNODEEXISTS) + { + throwNameCollisionCannotRename(new_entity->getType(), old_name, new_name); + } + else if (res == Coordination::Error::ZNONODE) + { + throwNotFound(id); + } + else + { + zkutil::KeeperMultiException::check(res, ops, responses); + } +} + + +void ReplicatedAccessStorage::runWorkerThread() +{ + LOG_DEBUG(getLogger(), "Started worker thread"); + while (!stop_flag) + { + try + { + if (!initialized) + initializeZookeeper(); + refresh(); + } + catch (...) + { + tryLogCurrentException(getLogger(), "Unexpected error, will try to restart worker thread:"); + resetAfterError(); + sleepForSeconds(5); + } + } +} + +void ReplicatedAccessStorage::resetAfterError() +{ + initialized = false; + + UUID id; + while (refresh_queue.tryPop(id)) {} + + std::lock_guard lock{mutex}; + for (const auto type : collections::range(EntityType::MAX)) + entries_by_name_and_type[static_cast(type)].clear(); + entries_by_id.clear(); +} + +void ReplicatedAccessStorage::initializeZookeeper() +{ + assert(!initialized); + auto zookeeper = get_zookeeper(); + + if (!zookeeper) + throw Exception("Can't have Replicated access without ZooKeeper", ErrorCodes::NO_ZOOKEEPER); + + createRootNodes(zookeeper); + + refreshEntities(zookeeper); + + initialized = true; +} + +void ReplicatedAccessStorage::createRootNodes(const zkutil::ZooKeeperPtr & zookeeper) +{ + zookeeper->createAncestors(zookeeper_path); + zookeeper->createIfNotExists(zookeeper_path, ""); + zookeeper->createIfNotExists(zookeeper_path + "/uuid", ""); + for (const auto type : collections::range(EntityType::MAX)) + { + /// Create a znode for each type of AccessEntity + const auto type_info = EntityTypeInfo::get(type); + zookeeper->createIfNotExists(zookeeper_path + "/" + type_info.unique_char, ""); + } +} + +void ReplicatedAccessStorage::refresh() +{ + UUID id; + if (refresh_queue.tryPop(id, /* timeout_ms: */ 10000)) + { + if (stop_flag) + return; + + auto zookeeper = get_zookeeper(); + + if (id == UUIDHelpers::Nil) + refreshEntities(zookeeper); + else + refreshEntity(zookeeper, id); + } +} + + +void ReplicatedAccessStorage::refreshEntities(const zkutil::ZooKeeperPtr & zookeeper) +{ + LOG_DEBUG(getLogger(), "Refreshing entities list"); + + const String zookeeper_uuids_path = zookeeper_path + "/uuid"; + auto watch_entities_list = [this](const Coordination::WatchResponse &) + { + refresh_queue.push(UUIDHelpers::Nil); + }; + Coordination::Stat stat; + const auto entity_uuid_strs = zookeeper->getChildrenWatch(zookeeper_uuids_path, &stat, watch_entities_list); + + std::unordered_set entity_uuids; + entity_uuids.reserve(entity_uuid_strs.size()); + for (const String & entity_uuid_str : entity_uuid_strs) + entity_uuids.insert(parseUUID(entity_uuid_str)); + + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + std::lock_guard lock{mutex}; + + std::vector entities_to_remove; + /// Locally remove entities that were removed from ZooKeeper + for (const auto & pair : entries_by_id) + { + const UUID & entity_uuid = pair.first; + if (!entity_uuids.contains(entity_uuid)) + entities_to_remove.push_back(entity_uuid); + } + for (const auto & entity_uuid : entities_to_remove) + removeEntityNoLock(entity_uuid, notifications); + + /// Locally add entities that were added to ZooKeeper + for (const auto & entity_uuid : entity_uuids) + { + const auto it = entries_by_id.find(entity_uuid); + if (it == entries_by_id.end()) + refreshEntityNoLock(zookeeper, entity_uuid, notifications); + } + + LOG_DEBUG(getLogger(), "Refreshing entities list finished"); +} + +void ReplicatedAccessStorage::refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id) +{ + Notifications notifications; + SCOPE_EXIT({ notify(notifications); }); + std::lock_guard lock{mutex}; + + refreshEntityNoLock(zookeeper, id, notifications); +} + +void ReplicatedAccessStorage::refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, Notifications & notifications) +{ + LOG_DEBUG(getLogger(), "Refreshing entity {}", toString(id)); + + const auto watch_entity = [this, id](const Coordination::WatchResponse & response) + { + if (response.type == Coordination::Event::CHANGED) + refresh_queue.push(id); + }; + Coordination::Stat entity_stat; + const String entity_path = zookeeper_path + "/uuid/" + toString(id); + String entity_definition; + const bool exists = zookeeper->tryGetWatch(entity_path, entity_definition, &entity_stat, watch_entity); + if (exists) + { + const AccessEntityPtr entity = deserializeAccessEntity(entity_definition, entity_path); + setEntityNoLock(id, entity, notifications); + } + else + { + removeEntityNoLock(id, notifications); + } +} + + +void ReplicatedAccessStorage::setEntityNoLock(const UUID & id, const AccessEntityPtr & entity, Notifications & notifications) +{ + LOG_DEBUG(getLogger(), "Setting id {} to entity named {}", toString(id), entity->getName()); + const EntityType type = entity->getType(); + const String & name = entity->getName(); + + /// If the type+name already exists and is a different entity, remove old entity + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + if (auto it = entries_by_name.find(name); it != entries_by_name.end() && it->second->id != id) + { + removeEntityNoLock(it->second->id, notifications); + } + + /// If the entity already exists under a different type+name, remove old type+name + if (auto it = entries_by_id.find(id); it != entries_by_id.end()) + { + const AccessEntityPtr & existing_entity = it->second.entity; + const EntityType existing_type = existing_entity->getType(); + const String & existing_name = existing_entity->getName(); + if (existing_type != type || existing_name != name) + { + auto & existing_entries_by_name = entries_by_name_and_type[static_cast(existing_type)]; + existing_entries_by_name.erase(existing_name); + } + } + + auto & entry = entries_by_id[id]; + entry.id = id; + entry.entity = entity; + entries_by_name[name] = &entry; + prepareNotifications(entry, false, notifications); +} + + +void ReplicatedAccessStorage::removeEntityNoLock(const UUID & id, Notifications & notifications) +{ + LOG_DEBUG(getLogger(), "Removing entity with id {}", toString(id)); + const auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) + { + LOG_DEBUG(getLogger(), "Id {} not found, ignoring removal", toString(id)); + return; + } + + const Entry & entry = it->second; + const EntityType type = entry.entity->getType(); + const String & name = entry.entity->getName(); + prepareNotifications(entry, true, notifications); + + auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + const auto name_it = entries_by_name.find(name); + if (name_it == entries_by_name.end()) + LOG_WARNING(getLogger(), "Entity {} not found in names, ignoring removal of name", toString(id)); + else if (name_it->second != &(it->second)) + LOG_WARNING(getLogger(), "Name {} not pointing to entity {}, ignoring removal of name", name, toString(id)); + else + entries_by_name.erase(name); + + entries_by_id.erase(id); + LOG_DEBUG(getLogger(), "Removed entity with id {}", toString(id)); +} + + +std::optional ReplicatedAccessStorage::findImpl(EntityType type, const String & name) const +{ + std::lock_guard lock{mutex}; + const auto & entries_by_name = entries_by_name_and_type[static_cast(type)]; + const auto it = entries_by_name.find(name); + if (it == entries_by_name.end()) + return {}; + + const Entry * entry = it->second; + return entry->id; +} + + +std::vector ReplicatedAccessStorage::findAllImpl(EntityType type) const +{ + std::lock_guard lock{mutex}; + std::vector result; + result.reserve(entries_by_id.size()); + for (const auto & [id, entry] : entries_by_id) + if (entry.entity->isTypeOf(type)) + result.emplace_back(id); + return result; +} + + +bool ReplicatedAccessStorage::existsImpl(const UUID & id) const +{ + std::lock_guard lock{mutex}; + return entries_by_id.count(id); +} + + +AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id) const +{ + std::lock_guard lock{mutex}; + const auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) + throwNotFound(id); + const Entry & entry = it->second; + return entry.entity; +} + + +String ReplicatedAccessStorage::readNameImpl(const UUID & id) const +{ + return readImpl(id)->getName(); +} + + +void ReplicatedAccessStorage::prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const +{ + const AccessEntityPtr entity = remove ? nullptr : entry.entity; + for (const auto & handler : entry.handlers_by_id) + notifications.push_back({handler, entry.id, entity}); + + for (const auto & handler : handlers_by_type[static_cast(entry.entity->getType())]) + notifications.push_back({handler, entry.id, entity}); +} + + +scope_guard ReplicatedAccessStorage::subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const +{ + std::lock_guard lock{mutex}; + auto & handlers = handlers_by_type[static_cast(type)]; + handlers.push_back(handler); + auto handler_it = std::prev(handlers.end()); + + return [this, type, handler_it] + { + std::lock_guard lock2{mutex}; + auto & handlers2 = handlers_by_type[static_cast(type)]; + handlers2.erase(handler_it); + }; +} + + +scope_guard ReplicatedAccessStorage::subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const +{ + std::lock_guard lock{mutex}; + const auto it = entries_by_id.find(id); + if (it == entries_by_id.end()) + return {}; + const Entry & entry = it->second; + auto handler_it = entry.handlers_by_id.insert(entry.handlers_by_id.end(), handler); + + return [this, id, handler_it] + { + std::lock_guard lock2{mutex}; + auto it2 = entries_by_id.find(id); + if (it2 != entries_by_id.end()) + { + const Entry & entry2 = it2->second; + entry2.handlers_by_id.erase(handler_it); + } + }; +} + + +bool ReplicatedAccessStorage::hasSubscriptionImpl(const UUID & id) const +{ + std::lock_guard lock{mutex}; + const auto & it = entries_by_id.find(id); + if (it != entries_by_id.end()) + { + const Entry & entry = it->second; + return !entry.handlers_by_id.empty(); + } + return false; +} + + +bool ReplicatedAccessStorage::hasSubscriptionImpl(EntityType type) const +{ + std::lock_guard lock{mutex}; + const auto & handlers = handlers_by_type[static_cast(type)]; + return !handlers.empty(); +} +} diff --git a/src/Access/ReplicatedAccessStorage.h b/src/Access/ReplicatedAccessStorage.h new file mode 100644 index 00000000000..377a82fd511 --- /dev/null +++ b/src/Access/ReplicatedAccessStorage.h @@ -0,0 +1,87 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ +/// Implementation of IAccessStorage which keeps all data in zookeeper. +class ReplicatedAccessStorage : public IAccessStorage +{ +public: + static constexpr char STORAGE_TYPE[] = "replicated"; + + ReplicatedAccessStorage(const String & storage_name, const String & zookeeper_path, zkutil::GetZooKeeper get_zookeeper); + virtual ~ReplicatedAccessStorage() override; + + const char * getStorageType() const override { return STORAGE_TYPE; } + + virtual void startup(); + virtual void shutdown(); + +private: + String zookeeper_path; + zkutil::GetZooKeeper get_zookeeper; + + std::atomic initialized = false; + std::atomic stop_flag = false; + ThreadFromGlobalPool worker_thread; + ThreadSafeQueue refresh_queue; + + UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override; + void removeImpl(const UUID & id) override; + void updateImpl(const UUID & id, const UpdateFunc & update_func) override; + + void insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists); + void removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id); + void updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func); + + void runWorkerThread(); + void resetAfterError(); + void initializeZookeeper(); + void createRootNodes(const zkutil::ZooKeeperPtr & zookeeper); + + void refresh(); + void refreshEntities(const zkutil::ZooKeeperPtr & zookeeper); + void refreshEntity(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id); + void refreshEntityNoLock(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, Notifications & notifications); + + void setEntityNoLock(const UUID & id, const AccessEntityPtr & entity, Notifications & notifications); + void removeEntityNoLock(const UUID & id, Notifications & notifications); + + struct Entry + { + UUID id; + AccessEntityPtr entity; + mutable std::list handlers_by_id; + }; + + std::optional findImpl(EntityType type, const String & name) const override; + std::vector findAllImpl(EntityType type) const override; + bool existsImpl(const UUID & id) const override; + AccessEntityPtr readImpl(const UUID & id) const override; + String readNameImpl(const UUID & id) const override; + bool canInsertImpl(const AccessEntityPtr &) const override { return true; } + + void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const; + scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override; + scope_guard subscribeForChangesImpl(EntityType type, const OnChangedHandler & handler) const override; + bool hasSubscriptionImpl(const UUID & id) const override; + bool hasSubscriptionImpl(EntityType type) const override; + + mutable std::mutex mutex; + std::unordered_map entries_by_id; + std::unordered_map entries_by_name_and_type[static_cast(EntityType::MAX)]; + mutable std::list handlers_by_type[static_cast(EntityType::MAX)]; +}; +} diff --git a/src/Access/ya.make b/src/Access/ya.make index 3ac4c944f2a..3bfc3c51eea 100644 --- a/src/Access/ya.make +++ b/src/Access/ya.make @@ -35,6 +35,7 @@ SRCS( Quota.cpp QuotaCache.cpp QuotaUsage.cpp + ReplicatedAccessStorage.cpp Role.cpp RoleCache.cpp RolesOrUsersSet.cpp diff --git a/tests/integration/test_replicated_users/__init__.py b/tests/integration/test_replicated_users/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_replicated_users/configs/config.xml b/tests/integration/test_replicated_users/configs/config.xml new file mode 100644 index 00000000000..1a9b8f9134c --- /dev/null +++ b/tests/integration/test_replicated_users/configs/config.xml @@ -0,0 +1,22 @@ + + + + + true + + node1 + 9000 + + + node2 + 9000 + + + + + + + /clickhouse/access + + + diff --git a/tests/integration/test_replicated_users/test.py b/tests/integration/test_replicated_users/test.py new file mode 100644 index 00000000000..75bc93921be --- /dev/null +++ b/tests/integration/test_replicated_users/test.py @@ -0,0 +1,73 @@ +import pytest + +from dataclasses import dataclass +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance('node1', main_configs=['configs/config.xml'], with_zookeeper=True, stay_alive=True) +node2 = cluster.add_instance('node2', main_configs=['configs/config.xml'], with_zookeeper=True, stay_alive=True) + +all_nodes = [node1, node2] + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@dataclass(frozen=True) +class Entity: + keyword: str + name: str + options: str = "" + + +entities = [ + Entity(keyword="USER", name="theuser"), + Entity(keyword="ROLE", name="therole"), + Entity(keyword="ROW POLICY", name="thepolicy", options=" ON default.t1"), + Entity(keyword="QUOTA", name="thequota"), + Entity(keyword="SETTINGS PROFILE", name="theprofile") +] + +def get_entity_id(entity): + return entity.keyword + + +@pytest.mark.parametrize("entity", entities, ids=get_entity_id) +def test_create_replicated(started_cluster, entity): + node1.query(f"CREATE {entity.keyword} {entity.name} {entity.options}") + assert f"cannot insert because {entity.keyword.lower()} `{entity.name}{entity.options}` already exists in replicated" in \ + node2.query_and_get_error(f"CREATE {entity.keyword} {entity.name} {entity.options}") + node1.query(f"DROP {entity.keyword} {entity.name} {entity.options}") + + +@pytest.mark.parametrize("entity", entities, ids=get_entity_id) +def test_create_and_delete_replicated(started_cluster, entity): + node1.query(f"CREATE {entity.keyword} {entity.name} {entity.options}") + node2.query(f"DROP {entity.keyword} {entity.name} {entity.options}") + + +@pytest.mark.parametrize("entity", entities, ids=get_entity_id) +def test_create_replicated_on_cluster(started_cluster, entity): + assert f"cannot insert because {entity.keyword.lower()} `{entity.name}{entity.options}` already exists in replicated" in \ + node1.query_and_get_error(f"CREATE {entity.keyword} {entity.name} ON CLUSTER default {entity.options}") + node1.query(f"DROP {entity.keyword} {entity.name} {entity.options}") + + +@pytest.mark.parametrize("entity", entities, ids=get_entity_id) +def test_create_replicated_if_not_exists_on_cluster(started_cluster, entity): + node1.query(f"CREATE {entity.keyword} IF NOT EXISTS {entity.name} ON CLUSTER default {entity.options}") + node1.query(f"DROP {entity.keyword} {entity.name} {entity.options}") + + +@pytest.mark.parametrize("entity", entities, ids=get_entity_id) +def test_rename_replicated(started_cluster, entity): + node1.query(f"CREATE {entity.keyword} {entity.name} {entity.options}") + node2.query(f"ALTER {entity.keyword} {entity.name} {entity.options} RENAME TO {entity.name}2") + node1.query(f"DROP {entity.keyword} {entity.name}2 {entity.options}") + From 21ddac09c653cf1b6dcff0a8818a97289b6241de Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Wed, 18 Aug 2021 02:01:49 +0300 Subject: [PATCH 159/220] Apply suggestions from code review Co-authored-by: Anna <42538400+adevyatova@users.noreply.github.com> --- docs/en/sql-reference/table-functions/cluster.md | 4 ++-- docs/ru/sql-reference/table-functions/cluster.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/en/sql-reference/table-functions/cluster.md b/docs/en/sql-reference/table-functions/cluster.md index d392cc80dc0..ff8422b6af9 100644 --- a/docs/en/sql-reference/table-functions/cluster.md +++ b/docs/en/sql-reference/table-functions/cluster.md @@ -7,7 +7,7 @@ toc_title: cluster Allows to access all shards in an existing cluster which configured in `remote_servers` section without creating a [Distributed](../../engines/table-engines/special/distributed.md) table. One replica of each shard is queried. -`clusterAllReplicas` function — same as `cluster`, but all replicas are queried. Each replica in a cluster is used as separate shard/connection. +`clusterAllReplicas` function — same as `cluster`, but all replicas are queried. Each replica in a cluster is used as a separate shard/connection. !!! note "Note" All available clusters are listed in the [system.clusters](../../operations/system-tables/clusters.md) table. @@ -24,7 +24,7 @@ clusterAllReplicas('cluster_name', db, table[, sharding_key]) - `cluster_name` – Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. - `db.table` or `db`, `table` - Name of a database and a table. -- `sharding_key` - When insert into cluster function with more than one shard, sharding key needs to be provided. Optional. +- `sharding_key` - A sharding key if the cluster has more than one shard. Optional. **Returned value** diff --git a/docs/ru/sql-reference/table-functions/cluster.md b/docs/ru/sql-reference/table-functions/cluster.md index e961e54dda4..b6727aebcca 100644 --- a/docs/ru/sql-reference/table-functions/cluster.md +++ b/docs/ru/sql-reference/table-functions/cluster.md @@ -32,7 +32,7 @@ clusterAllReplicas('cluster_name', db, table[, sharding_key]) **Использование макросов** -`cluster_name` может содержать макрос — подстановку в фигурных скобках. Эта подстановка заменяется на соответствующее значение из конфигурационного файла, из секции [macros](../../operations/server-configuration-parameters/settings.md#macros). +`cluster_name` может содержать макрос — подстановку в фигурных скобках. Эта подстановка заменяется на соответствующее значение из секции [macros](../../operations/server-configuration-parameters/settings.md#macros) конфигурационного файла . Пример: @@ -42,7 +42,7 @@ SELECT * FROM cluster('{cluster}', default.example_table); **Использование и рекомендации** -Использование табличных функций `cluster` и `clusterAllReplicas` менее оптимально, чем создание таблицы типа `Distributed`, поскольку в этом случае соединение с сервером переустанавливается на каждый запрос. При обработке большого количества запросов всегда создавайте `Distributed` таблицу заранее и не используйте табличные функции `cluster` и `clusterAllReplicas`. +Использование табличных функций `cluster` и `clusterAllReplicas` менее оптимально, чем создание таблицы типа `Distributed`, поскольку в этом случае при каждом новом запросе устанавливается новое соединение с сервером. При обработке большого количества запросов всегда создавайте `Distributed` таблицу заранее и не используйте табличные функции `cluster` и `clusterAllReplicas`. Табличные функции `cluster` and `clusterAllReplicas` могут быть полезны в следующих случаях: From f35b142e21d7a8301e470456e9d40d3376abd8a5 Mon Sep 17 00:00:00 2001 From: olgarev <56617294+olgarev@users.noreply.github.com> Date: Wed, 18 Aug 2021 02:03:46 +0300 Subject: [PATCH 160/220] Apply suggestions from code review --- docs/en/sql-reference/table-functions/cluster.md | 2 +- docs/ru/sql-reference/table-functions/cluster.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/en/sql-reference/table-functions/cluster.md b/docs/en/sql-reference/table-functions/cluster.md index ff8422b6af9..a02c2a10fb7 100644 --- a/docs/en/sql-reference/table-functions/cluster.md +++ b/docs/en/sql-reference/table-functions/cluster.md @@ -24,7 +24,7 @@ clusterAllReplicas('cluster_name', db, table[, sharding_key]) - `cluster_name` – Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers. - `db.table` or `db`, `table` - Name of a database and a table. -- `sharding_key` - A sharding key if the cluster has more than one shard. Optional. +- `sharding_key` - A sharding key. Optional. Needs to be specified if the cluster has more than one shard. **Returned value** diff --git a/docs/ru/sql-reference/table-functions/cluster.md b/docs/ru/sql-reference/table-functions/cluster.md index b6727aebcca..a9cff862293 100644 --- a/docs/ru/sql-reference/table-functions/cluster.md +++ b/docs/ru/sql-reference/table-functions/cluster.md @@ -24,7 +24,7 @@ clusterAllReplicas('cluster_name', db, table[, sharding_key]) - `cluster_name` – имя кластера, который обозначает подмножество адресов и параметров подключения к удаленным и локальным серверам, входящим в кластер. - `db.table` или `db`, `table` - имя базы данных и таблицы. -- `sharding_key` - ключ шардирования. Указывается, если данные добавляются более чем в один шард кластера. Необязательный аргумент. +- `sharding_key` - ключ шардирования. Необязательный аргумент. Указывается, если данные добавляются более чем в один шард кластера. **Возвращаемое значение** From 861776b8fe238caf58a4e9c08fca4fb2929a3039 Mon Sep 17 00:00:00 2001 From: zhangxiao871 Date: Wed, 18 Aug 2021 10:40:04 +0800 Subject: [PATCH 161/220] check genuine_throw and fake_throw are True. --- tests/integration/test_keeper_back_to_back/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index 48af4de4198..e0d86f05657 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -112,6 +112,7 @@ def test_sequential_nodes(started_cluster): except Exception as ex: fake_throw = True + assert genuine_throw == True assert genuine_throw == fake_throw genuine_childs_1 = list(sorted(genuine_zk.get_children("/test_sequential_nodes_1"))) From 5d3237c680023c27b0a1e53e3aef3379fb30dc16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B0=8F=E8=B7=AF?= <821008736@qq.com> Date: Wed, 18 Aug 2021 10:56:05 +0800 Subject: [PATCH 162/220] Update test.py --- tests/integration/test_keeper_back_to_back/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/test_keeper_back_to_back/test.py b/tests/integration/test_keeper_back_to_back/test.py index e0d86f05657..64f2f42d71e 100644 --- a/tests/integration/test_keeper_back_to_back/test.py +++ b/tests/integration/test_keeper_back_to_back/test.py @@ -113,7 +113,7 @@ def test_sequential_nodes(started_cluster): fake_throw = True assert genuine_throw == True - assert genuine_throw == fake_throw + assert fake_throw == True genuine_childs_1 = list(sorted(genuine_zk.get_children("/test_sequential_nodes_1"))) fake_childs_1 = list(sorted(fake_zk.get_children("/test_sequential_nodes_1"))) From 8199399159b34d3ce6421f7775113b316a4a64fb Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 18 Aug 2021 07:00:51 +0300 Subject: [PATCH 163/220] Update entrypoint.sh --- docker/test/performance-comparison/entrypoint.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 1d03d953ccd..406cb6d8f9d 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -127,6 +127,8 @@ export PATH export REF_PR export REF_SHA +ulimit -c unlimited + # Start the main comparison script. { \ time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \ From 3eb4b2ab7773caca7423d77280bd2bd4d01584df Mon Sep 17 00:00:00 2001 From: Nikita Mikhalev Date: Wed, 18 Aug 2021 10:23:41 +0500 Subject: [PATCH 164/220] Improved phrasing of attention about ALTER on Buffer table Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/ru/engines/table-engines/special/buffer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/special/buffer.md b/docs/ru/engines/table-engines/special/buffer.md index 3e3c9226933..a3ba9f85f05 100644 --- a/docs/ru/engines/table-engines/special/buffer.md +++ b/docs/ru/engines/table-engines/special/buffer.md @@ -51,7 +51,7 @@ CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10 Если есть необходимость выполнить ALTER для подчинённой таблицы и для таблицы Buffer, то рекомендуется удалить таблицу Buffer, затем выполнить ALTER подчинённой таблицы, а затем создать таблицу Buffer заново. !!! attention "Внимание" - В релизах до 28 сентября 2020 года выполнение ALTER на таблице Buffer ломает структуру блоков и вызывает ошибку (см. [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), поэтому удаление буфера и его создание заново - единственный вариант миграции для данного движка. Убедитесь, что в вашей версии ошибка устранена перед выполнением ALTER на таблице Buffer. + В релизах до 28 сентября 2020 года выполнение ALTER на таблице Buffer ломает структуру блоков и вызывает ошибку (см. [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), поэтому удаление буфера и его пересоздание — единственный вариант миграции для данного движка. Перед выполнением ALTER на таблице Buffer убедитесь, что в вашей версии эта ошибка устранена. При нештатном перезапуске сервера, данные, находящиеся в буфере, будут потеряны. From 62abef73f16a592d00058c80ea0589df7facb529 Mon Sep 17 00:00:00 2001 From: Nikita Mikhalev Date: Wed, 18 Aug 2021 10:24:42 +0500 Subject: [PATCH 165/220] Improved phrasing of attention about ALTER on Buffer table Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/en/engines/table-engines/special/buffer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index a31bb462754..0e7f0a53da8 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -57,7 +57,7 @@ The same thing happens if the subordinate table does not exist when the buffer i If you need to run ALTER for a subordinate table, and the Buffer table, we recommend first deleting the Buffer table, running ALTER for the subordinate table, then creating the Buffer table again. !!! attention "Attention" - Running ALTER on Buffer table in releases made prior to 28 Sep 2020 will cause `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), so deleting Buffer table and then recreating it is the only option. Please check error is fixed in your release before trying to run ALTER on Buffer table. + Running ALTER on the Buffer table in releases made before 28 Sep 2020 will cause a `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), so deleting the Buffer table and then recreating is the only option. It is advisable to check that this error is fixed in your release before trying to run ALTER on the Buffer table. If the server is restarted abnormally, the data in the buffer is lost. From fbc054f588eb6d98ae63c86bdfd942c80cb81ed3 Mon Sep 17 00:00:00 2001 From: Nikita Mikhalev Date: Wed, 18 Aug 2021 10:24:54 +0500 Subject: [PATCH 166/220] Improved phrasing of attention about ALTER on Buffer table Co-authored-by: Alexey Boykov <33257111+mathalex@users.noreply.github.com> --- docs/ru/engines/table-engines/special/buffer.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ru/engines/table-engines/special/buffer.md b/docs/ru/engines/table-engines/special/buffer.md index a3ba9f85f05..0c1ae591ae3 100644 --- a/docs/ru/engines/table-engines/special/buffer.md +++ b/docs/ru/engines/table-engines/special/buffer.md @@ -48,7 +48,7 @@ CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10 Если у одного из столбцов таблицы Buffer и подчинённой таблицы не совпадает тип, то в лог сервера будет записано сообщение об ошибке и буфер будет очищен. То же самое происходит, если подчинённая таблица не существует в момент сброса буфера. -Если есть необходимость выполнить ALTER для подчинённой таблицы и для таблицы Buffer, то рекомендуется удалить таблицу Buffer, затем выполнить ALTER подчинённой таблицы, а затем создать таблицу Buffer заново. +Если есть необходимость выполнить ALTER для подчинённой таблицы и для таблицы Buffer, то рекомендуется удалить таблицу Buffer, затем выполнить ALTER подчинённой таблицы, а после создать таблицу Buffer заново. !!! attention "Внимание" В релизах до 28 сентября 2020 года выполнение ALTER на таблице Buffer ломает структуру блоков и вызывает ошибку (см. [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), поэтому удаление буфера и его пересоздание — единственный вариант миграции для данного движка. Перед выполнением ALTER на таблице Buffer убедитесь, что в вашей версии эта ошибка устранена. From 788384be12b5e006a605bdffbcc266a63d0b7c2a Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 18 Aug 2021 09:47:58 +0300 Subject: [PATCH 167/220] Drop MALLOC_LIBARIRIES usage a) MALLOC_LIBARIRIES had been removed in 528e42bc4cd2ce7735040a45ea878de8a540c361 ("Improve jemalloc contrib") b) after JEMALLOC_LIBRARIES left, that had been removed in c160b251ba49ac89c1c49939d040fc2bbcb4cebe ("Drop support for unbundled jemalloc") in #15828 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 35c22526816..0599cf5a1e3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -593,7 +593,7 @@ macro (add_executable target) # disabled for TSAN and gcc since libtsan.a provides overrides too if (TARGET clickhouse_new_delete) # operator::new/delete for executables (MemoryTracker stuff) - target_link_libraries (${target} PRIVATE clickhouse_new_delete ${MALLOC_LIBRARIES}) + target_link_libraries (${target} PRIVATE clickhouse_new_delete) endif() endif() endmacro() From 342c3fae3ab84bf6e8b1d47c08d091dcdd3b8abb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 18 Aug 2021 09:47:59 +0300 Subject: [PATCH 168/220] Revert "Disable jemalloc under OSX" (only jemalloc hunk) This reverts commit f062aa8574d71146d293bc777d86aa2035b1fd38. --- contrib/jemalloc-cmake/CMakeLists.txt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 52208bb0278..9531a5a4f9e 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -1,9 +1,10 @@ -# Disabled under OSX until https://github.com/ClickHouse/ClickHouse/issues/27568 is fixed if (SANITIZE OR NOT ( - ((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_ARM OR ARCH_PPC64LE)))) + ((OS_LINUX OR OS_FREEBSD) AND (ARCH_AMD64 OR ARCH_ARM OR ARCH_PPC64LE)) OR + (OS_DARWIN AND (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo" OR CMAKE_BUILD_TYPE STREQUAL "Debug")) +)) if (ENABLE_JEMALLOC) message (${RECONFIGURE_MESSAGE_LEVEL} - "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64, aarch64, or ppc64le Linux or FreeBSD builds") + "jemalloc is disabled implicitly: it doesn't work with sanitizers and can only be used with x86_64, aarch64, or ppc64le Linux or FreeBSD builds and RelWithDebInfo macOS builds.") endif () set (ENABLE_JEMALLOC OFF) else () From 6a600e5f6ff80ad7e420223ada14432a7dcfe735 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 18 Aug 2021 09:47:59 +0300 Subject: [PATCH 169/220] jemalloc: fix under osx (zone_register() had been optimized out again) v2: use ld -u v3: move -u to executable --- CMakeLists.txt | 16 ++++++++++++++++ contrib/jemalloc-cmake/CMakeLists.txt | 4 ---- src/Common/memory.cpp | 25 ------------------------- 3 files changed, 16 insertions(+), 29 deletions(-) delete mode 100644 src/Common/memory.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 0599cf5a1e3..1aef8c9fc8d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -595,6 +595,22 @@ macro (add_executable target) # operator::new/delete for executables (MemoryTracker stuff) target_link_libraries (${target} PRIVATE clickhouse_new_delete) endif() + + # In case of static jemalloc, because zone_register() is located in zone.c and + # is never used outside (it is declared as constructor) it is omitted + # by the linker, and so jemalloc will not be registered as system + # allocator under osx [1], and clickhouse will SIGSEGV. + # + # [1]: https://github.com/jemalloc/jemalloc/issues/708 + # + # About symbol name: + # - _zone_register not zone_register due to Mach-O binary format, + # - _je_zone_register due to JEMALLOC_PRIVATE_NAMESPACE=je_ under OS X. + # - but jemalloc-cmake does not run private_namespace.sh + # so symbol name should be _zone_register + if (ENABLE_JEMALLOC AND MAKE_STATIC_LIBRARIES AND OS_DARWIN) + set_property(TARGET ${target} APPEND PROPERTY LINK_OPTIONS -u_zone_register) + endif() endif() endmacro() diff --git a/contrib/jemalloc-cmake/CMakeLists.txt b/contrib/jemalloc-cmake/CMakeLists.txt index 9531a5a4f9e..30dd3baa55b 100644 --- a/contrib/jemalloc-cmake/CMakeLists.txt +++ b/contrib/jemalloc-cmake/CMakeLists.txt @@ -139,9 +139,5 @@ target_compile_options(jemalloc PRIVATE -Wno-redundant-decls) target_compile_options(jemalloc PRIVATE -D_GNU_SOURCE) set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS USE_JEMALLOC=1) -if (MAKE_STATIC_LIBRARIES) - # To detect whether we need to register jemalloc for osx as default zone. - set_property(TARGET jemalloc APPEND PROPERTY INTERFACE_COMPILE_DEFINITIONS BUNDLED_STATIC_JEMALLOC=1) -endif() message (STATUS "Using jemalloc") diff --git a/src/Common/memory.cpp b/src/Common/memory.cpp deleted file mode 100644 index a79d3572071..00000000000 --- a/src/Common/memory.cpp +++ /dev/null @@ -1,25 +0,0 @@ -#if defined(OS_DARWIN) && defined(BUNDLED_STATIC_JEMALLOC) - -extern "C" -{ - extern void zone_register(); -} - -struct InitializeJemallocZoneAllocatorForOSX -{ - InitializeJemallocZoneAllocatorForOSX() - { - /// In case of OSX jemalloc register itself as a default zone allocator. - /// - /// But when you link statically then zone_register() will not be called, - /// and even will be optimized out: - /// - /// It is ok to call it twice (i.e. in case of shared libraries) - /// Since zone_register() is a no-op if the default zone is already replaced with something. - /// - /// https://github.com/jemalloc/jemalloc/issues/708 - zone_register(); - } -} initializeJemallocZoneAllocatorForOSX; - -#endif From e921cdbdbe7784a0dc55f2e6ca0674dfe683a42d Mon Sep 17 00:00:00 2001 From: igomac <714541080@qq.com> Date: Wed, 18 Aug 2021 15:12:32 +0800 Subject: [PATCH 170/220] Update backup.md add toc --- docs/zh/operations/backup.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/zh/operations/backup.md b/docs/zh/operations/backup.md index 75fbd83089a..dbaafc8b8a7 100644 --- a/docs/zh/operations/backup.md +++ b/docs/zh/operations/backup.md @@ -1,3 +1,7 @@ +--- +toc_priority: 49 +toc_title: "\u6570\u636E\u5907\u4EFD" +--- # 数据备份 {#data-backup} From 6c566a46b6e08ad7fb9c225fa5e3748bfbea3642 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Wed, 18 Aug 2021 10:21:41 +0300 Subject: [PATCH 171/220] Update 01822_short_circuit.reference (after merging #27680) --- .../0_stateless/01822_short_circuit.reference | 330 +++++++++--------- 1 file changed, 165 insertions(+), 165 deletions(-) diff --git a/tests/queries/0_stateless/01822_short_circuit.reference b/tests/queries/0_stateless/01822_short_circuit.reference index 204bcd0538e..0cebb9d4f75 100644 --- a/tests/queries/0_stateless/01822_short_circuit.reference +++ b/tests/queries/0_stateless/01822_short_circuit.reference @@ -999,246 +999,246 @@ 1970-01-02 23:13:20 1970-01-03 02:00:00 1970-01-03 04:46:40 -0.00000 -2.00000 -3.00000 -4.00000 -5.00000 -5.00000 -7.00000 -8.00000 -9.00000 -10.00000 -10.00000 -12.00000 -13.00000 -14.00000 -15.00000 -15.00000 -17.00000 -18.00000 -19.00000 -20.00000 -0.00000 +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 \N \N \N \N -5.00000 +5 \N \N \N \N -10.00000 +10 \N \N \N \N -15.00000 +15 \N \N \N \N \N -1.00000 -2.00000 -3.00000 -4.00000 +1 +2 +3 +4 \N -6.00000 -7.00000 -8.00000 -9.00000 +6 +7 +8 +9 \N -11.00000 -12.00000 -13.00000 -14.00000 +11 +12 +13 +14 \N -16.00000 -17.00000 -18.00000 -19.00000 -0.00000 -2.00000 -3.00000 -4.00000 -5.00000 -5.00000 -7.00000 -8.00000 -9.00000 -10.00000 -10.00000 -12.00000 -13.00000 -14.00000 -15.00000 -15.00000 -17.00000 -18.00000 -19.00000 -20.00000 -0.00000 +16 +17 +18 +19 +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 \N \N \N \N -5.00000 +5 \N \N \N \N -10.00000 +10 \N \N \N \N -15.00000 +15 \N \N \N \N \N -1.00000 -2.00000 -3.00000 -4.00000 +1 +2 +3 +4 \N -6.00000 -7.00000 -8.00000 -9.00000 +6 +7 +8 +9 \N -11.00000 -12.00000 -13.00000 -14.00000 +11 +12 +13 +14 \N -16.00000 -17.00000 -18.00000 -19.00000 -0.00000 -2.00000 -3.00000 -4.00000 -5.00000 -5.00000 -7.00000 -8.00000 -9.00000 -10.00000 -10.00000 -12.00000 -13.00000 -14.00000 -15.00000 -15.00000 -17.00000 -18.00000 -19.00000 -20.00000 -0.00000 +16 +17 +18 +19 +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 \N \N \N \N -5.00000 +5 \N \N \N \N -10.00000 +10 \N \N \N \N -15.00000 +15 \N \N \N \N \N -1.00000 -2.00000 -3.00000 -4.00000 +1 +2 +3 +4 \N -6.00000 -7.00000 -8.00000 -9.00000 +6 +7 +8 +9 \N -11.00000 -12.00000 -13.00000 -14.00000 +11 +12 +13 +14 \N -16.00000 -17.00000 -18.00000 -19.00000 -0.00000 -2.00000 -3.00000 -4.00000 -5.00000 -5.00000 -7.00000 -8.00000 -9.00000 -10.00000 -10.00000 -12.00000 -13.00000 -14.00000 -15.00000 -15.00000 -17.00000 -18.00000 -19.00000 -20.00000 -0.00000 +16 +17 +18 +19 +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 \N \N \N \N -5.00000 +5 \N \N \N \N -10.00000 +10 \N \N \N \N -15.00000 +15 \N \N \N \N \N -1.00000 -2.00000 -3.00000 -4.00000 +1 +2 +3 +4 \N -6.00000 -7.00000 -8.00000 -9.00000 +6 +7 +8 +9 \N -11.00000 -12.00000 -13.00000 -14.00000 +11 +12 +13 +14 \N -16.00000 -17.00000 -18.00000 -19.00000 +16 +17 +18 +19 [] [0,1] [0,1,2] @@ -1390,8 +1390,8 @@ Decimal32 21 14 10 -0.00 -42.00 -21.00 -14.00 -10.50 +0 +42 +21 +14 +10.5 From 523de98e2dc3f1418504489d6a9db139ff155fb1 Mon Sep 17 00:00:00 2001 From: Amos Bird Date: Wed, 18 Aug 2021 15:23:32 +0800 Subject: [PATCH 172/220] Proper shutdown global context --- src/Interpreters/Context.cpp | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 635af6f3cb7..bf9bd6409c4 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -359,7 +359,7 @@ struct ContextSharedPart String default_profile_name; /// Default profile name used for default values. String system_profile_name; /// Profile used by system processes String buffer_profile_name; /// Profile used by Buffer engine for flushing to the underlying - AccessControlManager access_control_manager; + std::unique_ptr access_control_manager; mutable UncompressedCachePtr uncompressed_cache; /// The cache of decompressed blocks. mutable MarkCachePtr mark_cache; /// Cache of marks in compressed files. mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads. @@ -419,7 +419,7 @@ struct ContextSharedPart Context::ConfigReloadCallback config_reload_callback; ContextSharedPart() - : macros(std::make_unique()) + : access_control_manager(std::make_unique()), macros(std::make_unique()) { /// TODO: make it singleton (?) static std::atomic num_calls{0}; @@ -498,6 +498,7 @@ struct ContextSharedPart distributed_schedule_pool.reset(); message_broker_schedule_pool.reset(); ddl_worker.reset(); + access_control_manager.reset(); /// Stop trace collector if any trace_collector.reset(); @@ -738,7 +739,7 @@ void Context::setConfig(const ConfigurationPtr & config) { auto lock = getLock(); shared->config = config; - shared->access_control_manager.setExternalAuthenticatorsConfig(*shared->config); + shared->access_control_manager->setExternalAuthenticatorsConfig(*shared->config); } const Poco::Util::AbstractConfiguration & Context::getConfigRef() const @@ -750,31 +751,31 @@ const Poco::Util::AbstractConfiguration & Context::getConfigRef() const AccessControlManager & Context::getAccessControlManager() { - return shared->access_control_manager; + return *shared->access_control_manager; } const AccessControlManager & Context::getAccessControlManager() const { - return shared->access_control_manager; + return *shared->access_control_manager; } void Context::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config) { auto lock = getLock(); - shared->access_control_manager.setExternalAuthenticatorsConfig(config); + shared->access_control_manager->setExternalAuthenticatorsConfig(config); } std::unique_ptr Context::makeGSSAcceptorContext() const { auto lock = getLock(); - return std::make_unique(shared->access_control_manager.getExternalAuthenticators().getKerberosParams()); + return std::make_unique(shared->access_control_manager->getExternalAuthenticators().getKerberosParams()); } void Context::setUsersConfig(const ConfigurationPtr & config) { auto lock = getLock(); shared->users_config = config; - shared->access_control_manager.setUsersConfig(*shared->users_config); + shared->access_control_manager->setUsersConfig(*shared->users_config); } ConfigurationPtr Context::getUsersConfig() From 59e3cb18f4e53c453951267b5599afeb664290d8 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Wed, 18 Aug 2021 10:58:21 +0300 Subject: [PATCH 173/220] Update entrypoint.sh --- docker/test/performance-comparison/entrypoint.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 406cb6d8f9d..19af56e3299 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -129,6 +129,7 @@ export REF_SHA ulimit -c unlimited + # Start the main comparison script. { \ time ../download.sh "$REF_PR" "$REF_SHA" "$PR_TO_TEST" "$SHA_TO_TEST" && \ From ca67bf0e5a4e54ade19765a76f5c7aceaa4acb69 Mon Sep 17 00:00:00 2001 From: vdimir Date: Wed, 18 Aug 2021 11:18:33 +0300 Subject: [PATCH 174/220] Fix NotJoinedBlocks::read, add logging --- src/Interpreters/HashJoin.cpp | 7 +++++++ src/Interpreters/join_common.cpp | 29 +++++++++++++++++------------ 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/src/Interpreters/HashJoin.cpp b/src/Interpreters/HashJoin.cpp index 6abaddd6270..46f8c9aac79 100644 --- a/src/Interpreters/HashJoin.cpp +++ b/src/Interpreters/HashJoin.cpp @@ -193,6 +193,13 @@ HashJoin::HashJoin(std::shared_ptr table_join_, const Block & right_s required_right_keys = table_join->getRequiredRightKeys(right_table_keys, required_right_keys_sources); + LOG_DEBUG(log, "Right keys: [{}] (required: [{}]), left keys: [{}]", + fmt::join(key_names_right, ", "), + fmt::join(required_right_keys.getNames(), ", "), + fmt::join(table_join->keyNamesLeft(), ", ")); + + LOG_DEBUG(log, "Columns to add: [{}]", sample_block_with_columns_to_add.dumpStructure()); + std::tie(condition_mask_column_name_left, condition_mask_column_name_right) = table_join->joinConditionColumnNames(); JoinCommon::removeLowCardinalityInplace(right_table_keys); diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index 349ba56e74a..b230d8d1957 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -500,6 +500,9 @@ NotJoinedBlocks::NotJoinedBlocks(std::unique_ptr filler_, , saved_block_sample(filler->getEmptyBlock()) , result_sample_block(materializeBlock(result_sample_block_)) { + LOG_DEBUG(&Poco::Logger::get("NotJoinedBlocks"), "saved_block_sample {}",saved_block_sample.dumpStructure()); + LOG_DEBUG(&Poco::Logger::get("NotJoinedBlocks"), "result_sample_block {}",result_sample_block.dumpStructure()); + for (size_t left_pos = 0; left_pos < left_columns_count; ++left_pos) { /// We need right 'x' for 'RIGHT JOIN ... USING(x)' @@ -621,23 +624,25 @@ void NotJoinedBlocks::copySameKeys(Block & block) const } Block NotJoinedBlocks::read() - { - Block right_block = filler->getEmptyBlock(); - MutableColumns columns_right = right_block.cloneEmptyColumns(); - size_t rows_added = filler->fillColumns(columns_right); - if (rows_added == 0) - return {}; + Block result_block = result_sample_block.cloneEmpty(); + { + Block right_block = filler->getEmptyBlock(); + MutableColumns columns_right = right_block.cloneEmptyColumns(); + size_t rows_added = filler->fillColumns(columns_right); + if (rows_added == 0) + return {}; - addLeftColumns(right_block, rows_added); - addRightColumns(right_block, columns_right); - copySameKeys(right_block); - correctLowcardAndNullability(right_block); + addLeftColumns(result_block, rows_added); + addRightColumns(result_block, columns_right); + } + copySameKeys(result_block); + correctLowcardAndNullability(result_block); #ifndef NDEBUG - assertBlocksHaveEqualStructure(right_block, result_sample_block, "NotJoinedBlocks"); + assertBlocksHaveEqualStructure(result_block, result_sample_block, "NotJoinedBlocks"); #endif - return right_block; + return result_block; } } From 5cc98c67a476325faaa80875ec5a6e356c15358d Mon Sep 17 00:00:00 2001 From: alesapin Date: Wed, 18 Aug 2021 11:56:18 +0300 Subject: [PATCH 175/220] Fix --- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- src/Storages/MergeTree/MergeTreePartInfo.h | 6 ++++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 37309cec5b6..34c3276c7f5 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -2395,7 +2395,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c /// It's a DROP PART and it's already executed by fetching some covering part bool is_drop_part = !drop_range.isFakeDropRangePart() && drop_range.min_block; - if (is_drop_part && (part->info.min_block != drop_range.min_block || part->info.max_block != drop_range.max_block || part->info.getDataVersion() != drop_range.getDataVersion())) + if (is_drop_part && (part->info.min_block != drop_range.min_block || part->info.max_block != drop_range.max_block || part->info.getMutationVersion() != drop_range.getMutationVersion())) { /// Why we check only min and max blocks here without checking merge /// level? It's a tricky situation which can happen on a stale @@ -2412,7 +2412,7 @@ MergeTreeData::DataPartsVector MergeTreeData::removePartsInRangeFromWorkingSet(c /// So here we just check that all_1_3_1 covers blocks from drop /// all_2_2_2. /// - bool is_covered_by_min_max_block = part->info.min_block <= drop_range.min_block && part->info.max_block >= drop_range.max_block && part->info.getDataVersion() >= drop_range.getDataVersion(); + bool is_covered_by_min_max_block = part->info.min_block <= drop_range.min_block && part->info.max_block >= drop_range.max_block && part->info.getMutationVersion() >= drop_range.getMutationVersion(); if (is_covered_by_min_max_block) { LOG_INFO(log, "Skipping drop range for part {} because covering part {} already exists", drop_range.getPartName(), part->name); diff --git a/src/Storages/MergeTree/MergeTreePartInfo.h b/src/Storages/MergeTree/MergeTreePartInfo.h index be856c1f157..181fef7990c 100644 --- a/src/Storages/MergeTree/MergeTreePartInfo.h +++ b/src/Storages/MergeTree/MergeTreePartInfo.h @@ -65,6 +65,12 @@ struct MergeTreePartInfo && mutation >= rhs.mutation; } + /// Return part mutation version, if part wasn't mutated return zero + Int64 getMutationVersion() const + { + return mutation ? mutation : 0; + } + /// True if parts do not intersect in any way. bool isDisjoint(const MergeTreePartInfo & rhs) const { From 05d77d2873bf8c79c28757bccbc06bbdb491036e Mon Sep 17 00:00:00 2001 From: Mikhail Filimonov Date: Tue, 17 Aug 2021 19:12:12 +0200 Subject: [PATCH 176/220] Safer processing of NULL messages in Kafka for some formats --- src/Storages/Kafka/KafkaBlockInputStream.cpp | 6 +++++- .../Kafka/ReadBufferFromKafkaConsumer.cpp | 15 +++++++++++---- tests/integration/test_storage_kafka/test.py | 5 +++++ 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/src/Storages/Kafka/KafkaBlockInputStream.cpp b/src/Storages/Kafka/KafkaBlockInputStream.cpp index 5d9b19b1972..95fa1459e76 100644 --- a/src/Storages/Kafka/KafkaBlockInputStream.cpp +++ b/src/Storages/Kafka/KafkaBlockInputStream.cpp @@ -252,7 +252,11 @@ Block KafkaBlockInputStream::readImpl() } else { - LOG_WARNING(log, "Parsing of message (topic: {}, partition: {}, offset: {}) return no rows.", buffer->currentTopic(), buffer->currentPartition(), buffer->currentOffset()); + // We came here in case of tombstone (or sometimes zero-length) messages, and it is not something abnormal + // TODO: it seems like in case of put_error_to_stream=true we may need to process those differently + // currently we just skip them with note in logs. + buffer->storeLastReadMessageOffset(); + LOG_DEBUG(log, "Parsing of message (topic: {}, partition: {}, offset: {}) return no rows.", buffer->currentTopic(), buffer->currentPartition(), buffer->currentOffset()); } if (!buffer->hasMorePolledMessages() diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index bd25607a5f3..f5f1974dcfe 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -466,13 +466,20 @@ bool ReadBufferFromKafkaConsumer::nextImpl() if (!allowed || !hasMorePolledMessages()) return false; - // XXX: very fishy place with const casting. - auto * new_position = reinterpret_cast(const_cast(current->get_payload().get_data())); - BufferBase::set(new_position, current->get_payload().get_size(), 0); - allowed = false; + const auto * message_data = current->get_payload().get_data(); + size_t message_size = current->get_payload().get_size(); + allowed = false; ++current; + // in some cases message can be NULL (tombstone records for example) + // parsers are not ready to get NULLs on input. + if (unlikely(message_data == nullptr)) + return false; + + // XXX: very fishy place with const casting. + auto * new_position = reinterpret_cast(const_cast(message_data)); + BufferBase::set(new_position, message_size, 0); return true; } diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index cff2b972983..21d6c7c10ab 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -283,6 +283,11 @@ def test_kafka_json_as_string(kafka_cluster): kafka_produce(kafka_cluster, 'kafka_json_as_string', ['{"t": 123, "e": {"x": "woof"} }', '', '{"t": 124, "e": {"x": "test"} }', '{"F1":"V1","F2":{"F21":"V21","F22":{},"F23":"V23","F24":"2019-12-24T16:28:04"},"F3":"V3"}']) + # 'tombstone' record (null value) = marker of deleted record + producer = KafkaProducer(bootstrap_servers="localhost:{}".format(cluster.kafka_port), value_serializer=producer_serializer, key_serializer=producer_serializer) + producer.send(topic='kafka_json_as_string', key='xxx') + producer.flush() + instance.query(''' CREATE TABLE test.kafka (field String) ENGINE = Kafka From 09ff66da0e026b5b1f6352e438fb0b7b7ae7d1dd Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 18 Aug 2021 12:49:22 +0300 Subject: [PATCH 177/220] fix a couple of bugs that may cause replicas to diverge --- .../ReplicatedMergeTreePartCheckThread.cpp | 1 + .../MergeTree/ReplicatedMergeTreeQueue.cpp | 27 ++++++++-- .../MergeTree/ReplicatedMergeTreeQueue.h | 11 +++- .../ReplicatedMergeTreeRestartingThread.cpp | 23 ++++++-- src/Storages/StorageReplicatedMergeTree.cpp | 53 ++++++++++--------- 5 files changed, 83 insertions(+), 32 deletions(-) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 35a011a4a58..797d0570fbc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -145,6 +145,7 @@ ReplicatedMergeTreePartCheckThread::MissingPartSearchResult ReplicatedMergeTreeP if (found_part_with_the_same_min_block && found_part_with_the_same_max_block) { + /// FIXME It may never appear LOG_WARNING(log, "Found parts with the same min block and with the same max block as the missing part {}. Hoping that it will eventually appear as a result of a merge.", part_name); return MissingPartSearchResult::FoundAndDontNeedFetch; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index ef276a53df2..c71a79d2009 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int UNEXPECTED_NODE_IN_ZOOKEEPER; extern const int ABORTED; + extern const int READONLY; } @@ -472,9 +473,15 @@ bool ReplicatedMergeTreeQueue::removeFailedQuorumPart(const MergeTreePartInfo & return virtual_parts.remove(part_info); } -int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback) +int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback, PullLogsReason reason) { std::lock_guard lock(pull_logs_to_queue_mutex); + if (storage.is_readonly && reason == SYNC) + { + throw Exception(ErrorCodes::READONLY, "Cannot SYNC REPLICA, because replica is readonly"); + /// TODO throw logical error for other reasons (except LOAD) + } + if (pull_log_blocker.isCancelled()) throw Exception("Log pulling is cancelled", ErrorCodes::ABORTED); @@ -714,13 +721,22 @@ void ReplicatedMergeTreeQueue::updateMutations(zkutil::ZooKeeperPtr zookeeper, C std::vector> futures; for (const String & entry : entries_to_load) - futures.emplace_back(zookeeper->asyncGet(fs::path(zookeeper_path) / "mutations" / entry)); + futures.emplace_back(zookeeper->asyncTryGet(fs::path(zookeeper_path) / "mutations" / entry)); std::vector new_mutations; for (size_t i = 0; i < entries_to_load.size(); ++i) { + auto maybe_response = futures[i].get(); + if (maybe_response.error != Coordination::Error::ZOK) + { + assert(maybe_response.error == Coordination::Error::ZNONODE); + /// It's ok if it happened on server startup or table creation and replica loads all mutation entries. + /// It's also ok if mutation was killed. + LOG_WARNING(log, "Cannot get mutation node {} ({}), probably it was concurrently removed", entries_to_load[i], maybe_response.error); + continue; + } new_mutations.push_back(std::make_shared( - ReplicatedMergeTreeMutationEntry::parse(futures[i].get().data, entries_to_load[i]))); + ReplicatedMergeTreeMutationEntry::parse(maybe_response.data, entries_to_load[i]))); } bool some_mutations_are_probably_done = false; @@ -1504,6 +1520,9 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( /// to allow recovering from a mutation that cannot be executed. This way you can delete the mutation entry /// from /mutations in ZK and the replicas will simply skip the mutation. + /// NOTE: However, it's quite dangerous to skip MUTATE_PART. Replicas may diverge if one of them have executed part mutation, + /// and then mutation was killed before execution of MUTATE_PART on remaining replicas. + if (part->info.getDataVersion() > desired_mutation_version) { LOG_WARNING(log, "Data version of part {} is already greater than desired mutation version {}", part->name, desired_mutation_version); @@ -1831,7 +1850,7 @@ ReplicatedMergeTreeMergePredicate::ReplicatedMergeTreeMergePredicate( } } - merges_version = queue_.pullLogsToQueue(zookeeper); + merges_version = queue_.pullLogsToQueue(zookeeper, {}, ReplicatedMergeTreeQueue::MERGE_PREDICATE); { /// We avoid returning here a version to be used in a lightweight transaction. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h index e49d80fc832..57e1e658665 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.h @@ -294,13 +294,22 @@ public: bool removeFailedQuorumPart(const MergeTreePartInfo & part_info); + enum PullLogsReason + { + LOAD, + UPDATE, + MERGE_PREDICATE, + SYNC, + OTHER, + }; + /** Copy the new entries from the shared log to the queue of this replica. Set the log_pointer to the appropriate value. * If watch_callback is not empty, will call it when new entries appear in the log. * If there were new entries, notifies storage.queue_task_handle. * Additionally loads mutations (so that the set of mutations is always more recent than the queue). * Return the version of "logs" node (that is updated for every merge/mutation/... added to the log) */ - int32_t pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback = {}); + int32_t pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback = {}, PullLogsReason reason = OTHER); /// Load new mutation entries. If something new is loaded, schedule storage.merge_selecting_task. /// If watch_callback is not empty, will call it when new mutations appear in ZK. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 25f25480549..a7bb56f1955 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -25,6 +25,8 @@ namespace DB namespace ErrorCodes { extern const int REPLICA_IS_ALREADY_ACTIVE; + extern const int REPLICA_STATUS_CHANGED; + } namespace @@ -55,6 +57,7 @@ void ReplicatedMergeTreeRestartingThread::run() if (need_stop) return; + bool reschedule_now = false; try { if (first_time || readonly_mode_was_set || storage.getZooKeeper()->expired()) @@ -131,15 +134,29 @@ void ReplicatedMergeTreeRestartingThread::run() first_time = false; } } - catch (...) + catch (const Exception & e) { /// We couldn't activate table let's set it into readonly mode setReadonly(); + partialShutdown(); + storage.startup_event.set(); + tryLogCurrentException(log, __PRETTY_FUNCTION__); + + if (e.code() == ErrorCodes::REPLICA_STATUS_CHANGED) + reschedule_now = true; + } + catch (...) + { + setReadonly(); + partialShutdown(); storage.startup_event.set(); tryLogCurrentException(log, __PRETTY_FUNCTION__); } - task->scheduleAfter(check_period_ms); + if (reschedule_now) + task->schedule(); + else + task->scheduleAfter(check_period_ms); } @@ -159,7 +176,7 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() /// pullLogsToQueue() after we mark replica 'is_active' (and after we repair if it was lost); /// because cleanup_thread doesn't delete log_pointer of active replicas. - storage.queue.pullLogsToQueue(zookeeper); + storage.queue.pullLogsToQueue(zookeeper, {}, ReplicatedMergeTreeQueue::LOAD); storage.queue.removeCurrentPartsFromMutations(); storage.last_queue_update_finish_time.store(time(nullptr)); diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 150a71a09e5..bdec69095ce 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -141,6 +141,7 @@ namespace ErrorCodes extern const int DUPLICATE_DATA_PART; extern const int BAD_ARGUMENTS; extern const int CONCURRENT_ACCESS_NOT_SUPPORTED; + extern const int CHECKSUM_DOESNT_MATCH; } namespace ActionLocks @@ -1314,32 +1315,35 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: } ReplicatedMergeTreePartHeader replica_part_header; - if (!part_zk_str.empty()) - replica_part_header = ReplicatedMergeTreePartHeader::fromString(part_zk_str); - else + if (part_zk_str.empty()) { - Coordination::Stat columns_stat_before, columns_stat_after; String columns_str; String checksums_str; - /// Let's check that the node's version with the columns did not change while we were reading the checksums. - /// This ensures that the columns and the checksum refer to the same - if (!zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str, &columns_stat_before) || - !zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str) || - !zookeeper->exists(fs::path(current_part_path) / "columns", &columns_stat_after) || - columns_stat_before.version != columns_stat_after.version) + if (zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str) && + zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str)) { - LOG_INFO(log, "Not checking checksums of part {} with replica {} because part changed while we were reading its checksums", part_name, replica); + replica_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksumsZNodes(columns_str, checksums_str); + } + else + { + if (zookeeper->exists(current_part_path)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} has empty header and does not have columns and checksums. " + "Looks like a bug.", current_part_path); + LOG_INFO(log, "Not checking checksums of part {} with replica {} because part was removed from ZooKeeper", part_name, replica); continue; } - - replica_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksumsZNodes( - columns_str, checksums_str); + } + else + { + replica_part_header = ReplicatedMergeTreePartHeader::fromString(part_zk_str); } if (replica_part_header.getColumnsHash() != local_part_header.getColumnsHash()) { - LOG_INFO(log, "Not checking checksums of part {} with replica {} because columns are different", part_name, replica); - continue; + /// Either it's a bug or ZooKeeper contains broken data. + /// TODO Fix KILL MUTATION and replace CHECKSUM_DOESNT_MATCH with LOGICAL_ERROR + /// (some replicas may skip killed mutation even if it was executed on other replicas) + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, "Part {} from {} has different columns hash", part_name, replica); } replica_part_header.getChecksums().checkEqual(local_part_header.getChecksums(), true); @@ -2137,6 +2141,8 @@ bool StorageReplicatedMergeTree::executeFetch(LogEntry & entry) if (!parts_for_merge.empty() && replica.empty()) { LOG_INFO(log, "No active replica has part {}. Will fetch merged part instead.", entry.new_part_name); + /// We should enqueue it for check, because merged part may never appear if source part is lost + enqueuePartForCheck(entry.new_part_name); return false; } @@ -3083,7 +3089,7 @@ void StorageReplicatedMergeTree::queueUpdatingTask() } try { - queue.pullLogsToQueue(getZooKeeper(), queue_updating_task->getWatchCallback()); + queue.pullLogsToQueue(getZooKeeper(), queue_updating_task->getWatchCallback(), ReplicatedMergeTreeQueue::UPDATE); last_queue_update_finish_time.store(time(nullptr)); queue_update_in_progress = false; } @@ -4319,11 +4325,9 @@ void StorageReplicatedMergeTree::startup() restarting_thread.start(); /// Wait while restarting_thread initializes LeaderElection (and so on) or makes first attempt to do it + /// TODO Do we still need startup_event? startup_event.wait(); - /// If we don't separate create/start steps, race condition will happen - /// between the assignment of queue_task_handle and queueTask that use the queue_task_handle. - background_executor.start(); startBackgroundMovesIfNeeded(); part_moves_between_shards_orchestrator.start(); @@ -5460,9 +5464,9 @@ bool StorageReplicatedMergeTree::waitForTableReplicaToProcessLogEntry( const auto & stop_waiting = [&]() { - bool stop_waiting_itself = waiting_itself && (partial_shutdown_called || is_dropped); + bool stop_waiting_itself = waiting_itself && partial_shutdown_called; bool stop_waiting_non_active = !wait_for_non_active && !getZooKeeper()->exists(fs::path(table_zookeeper_path) / "replicas" / replica / "is_active"); - return stop_waiting_itself || stop_waiting_non_active; + return is_dropped || stop_waiting_itself || stop_waiting_non_active; }; /// Don't recheck ZooKeeper too often @@ -6058,7 +6062,7 @@ CancellationCode StorageReplicatedMergeTree::killMutation(const String & mutatio zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - LOG_TRACE(log, "Killing mutation {}", mutation_id); + LOG_INFO(log, "Killing mutation {}", mutation_id); auto mutation_entry = queue.removeMutation(zookeeper, mutation_id); if (!mutation_entry) @@ -6964,7 +6968,7 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI Stopwatch watch; /// Let's fetch new log entries firstly - queue.pullLogsToQueue(getZooKeeper()); + queue.pullLogsToQueue(getZooKeeper(), {}, ReplicatedMergeTreeQueue::SYNC); /// This is significant, because the execution of this task could be delayed at BackgroundPool. /// And we force it to be executed. @@ -7202,6 +7206,7 @@ MutationCommands StorageReplicatedMergeTree::getFirstAlterMutationCommandsForPar void StorageReplicatedMergeTree::startBackgroundMovesIfNeeded() { + /// FIXME is it related to replication somehow? If it is we should start it from RestartingThread only if (areBackgroundMovesNeeded()) background_moves_executor.start(); } From d9e3adf3d5e5914f6080ef8328241311a8f2ef97 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 18 Aug 2021 12:14:00 +0200 Subject: [PATCH 178/220] 01766: Use a date without timezone changes --- .../0_stateless/01766_todatetime64_no_timezone_arg.reference | 2 +- .../queries/0_stateless/01766_todatetime64_no_timezone_arg.sql | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.reference b/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.reference index 52eea094ae4..b0e96ac9e54 100644 --- a/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.reference +++ b/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.reference @@ -1 +1 @@ -2021-03-22 00:00:00.000 +2021-03-23 00:00:00.000 diff --git a/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.sql b/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.sql index 99141a694c1..2aac922487e 100644 --- a/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.sql +++ b/tests/queries/0_stateless/01766_todatetime64_no_timezone_arg.sql @@ -1 +1 @@ -SELECT toDateTime64('2021-03-22', 3); +SELECT toDateTime64('2021-03-23', 3); From a46fe11e2cad62c7d6330cc086d5ce25e2e2fa90 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 18 Aug 2021 13:30:02 +0300 Subject: [PATCH 179/220] Try fix test_mutations_with_merge_background_task --- src/Functions/in.cpp | 10 +++---- src/Interpreters/ActionsVisitor.cpp | 4 +-- src/Interpreters/ExpressionAnalyzer.cpp | 4 +-- src/Interpreters/Set.cpp | 26 +++++++++---------- src/Interpreters/Set.h | 6 ++--- .../Transforms/CreatingSetsTransform.cpp | 4 +-- src/Storages/StorageSet.cpp | 6 ++--- .../System/StorageSystemZooKeeper.cpp | 4 +-- 8 files changed, 32 insertions(+), 32 deletions(-) diff --git a/src/Functions/in.cpp b/src/Functions/in.cpp index 17ec2518490..db4194308fc 100644 --- a/src/Functions/in.cpp +++ b/src/Functions/in.cpp @@ -102,7 +102,7 @@ public: throw Exception("Second argument for function '" + getName() + "' must be Set; found " + column_set_ptr->getName(), ErrorCodes::ILLEGAL_COLUMN); - Block columns_of_key_columns; + ColumnsWithTypeAndName columns_of_key_columns; /// First argument may be a tuple or a single column. const ColumnWithTypeAndName & left_arg = arguments[0]; @@ -125,16 +125,16 @@ public: const DataTypes & tuple_types = type_tuple->getElements(); size_t tuple_size = tuple_columns.size(); for (size_t i = 0; i < tuple_size; ++i) - columns_of_key_columns.insert({ tuple_columns[i], tuple_types[i], "_" + toString(i) }); + columns_of_key_columns.emplace_back(tuple_columns[i], tuple_types[i], "_" + toString(i)); } else - columns_of_key_columns.insert(left_arg); + columns_of_key_columns.emplace_back(left_arg); /// Replace single LowCardinality column to it's dictionary if possible. ColumnPtr lc_indexes = nullptr; - if (columns_of_key_columns.columns() == 1) + if (columns_of_key_columns.size() == 1) { - auto & arg = columns_of_key_columns.safeGetByPosition(0); + auto & arg = columns_of_key_columns.at(0); const auto * col = arg.column.get(); if (const auto * const_col = typeid_cast(col)) col = &const_col->getDataColumn(); diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 61e484ff6f1..9a27043160f 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -374,8 +374,8 @@ SetPtr makeExplicitSet( SetPtr set = std::make_shared(size_limits, create_ordered_set, context->getSettingsRef().transform_null_in); - set->setHeader(block.cloneEmpty()); - set->insertFromBlock(block); + set->setHeader(block.cloneEmpty().getColumnsWithTypeAndName()); + set->insertFromBlock(block.getColumnsWithTypeAndName()); set->finishInsert(); prepared_sets[set_key] = set; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index c8073cd92ad..c8a5ed6c56a 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -337,7 +337,7 @@ void ExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_ PullingAsyncPipelineExecutor executor(io.pipeline); SetPtr set = std::make_shared(settings.size_limits_for_set, true, getContext()->getSettingsRef().transform_null_in); - set->setHeader(executor.getHeader()); + set->setHeader(executor.getHeader().getColumnsWithTypeAndName()); Block block; while (executor.pull(block)) @@ -346,7 +346,7 @@ void ExpressionAnalyzer::tryMakeSetForIndexFromSubquery(const ASTPtr & subquery_ continue; /// If the limits have been exceeded, give up and let the default subquery processing actions take place. - if (!set->insertFromBlock(block)) + if (!set->insertFromBlock(block.getColumnsWithTypeAndName())) return; } diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 8202c1ccce2..5ab59ba3f07 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -99,14 +99,14 @@ void NO_INLINE Set::insertFromBlockImplCase( } -void Set::setHeader(const Block & header) +void Set::setHeader(const ColumnsWithTypeAndName & header) { std::unique_lock lock(rwlock); if (!data.empty()) return; - keys_size = header.columns(); + keys_size = header.size(); ColumnRawPtrs key_columns; key_columns.reserve(keys_size); data_types.reserve(keys_size); @@ -118,10 +118,10 @@ void Set::setHeader(const Block & header) /// Remember the columns we will work with for (size_t i = 0; i < keys_size; ++i) { - materialized_columns.emplace_back(header.safeGetByPosition(i).column->convertToFullColumnIfConst()); + materialized_columns.emplace_back(header.at(i).column->convertToFullColumnIfConst()); key_columns.emplace_back(materialized_columns.back().get()); - data_types.emplace_back(header.safeGetByPosition(i).type); - set_elements_types.emplace_back(header.safeGetByPosition(i).type); + data_types.emplace_back(header.at(i).type); + set_elements_types.emplace_back(header.at(i).type); /// Convert low cardinality column to full. if (const auto * low_cardinality_type = typeid_cast(data_types.back().get())) @@ -161,7 +161,7 @@ void Set::setHeader(const Block & header) } -bool Set::insertFromBlock(const Block & block) +bool Set::insertFromBlock(const ColumnsWithTypeAndName & columns) { std::unique_lock lock(rwlock); @@ -177,11 +177,11 @@ bool Set::insertFromBlock(const Block & block) /// Remember the columns we will work with for (size_t i = 0; i < keys_size; ++i) { - materialized_columns.emplace_back(block.safeGetByPosition(i).column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality()); + materialized_columns.emplace_back(columns.at(i).column->convertToFullColumnIfConst()->convertToFullColumnIfLowCardinality()); key_columns.emplace_back(materialized_columns.back().get()); } - size_t rows = block.rows(); + size_t rows = columns.at(0).column->size(); /// We will insert to the Set only keys, where all components are not NULL. ConstNullMapPtr null_map{}; @@ -192,7 +192,7 @@ bool Set::insertFromBlock(const Block & block) /// Filter to extract distinct values from the block. ColumnUInt8::MutablePtr filter; if (fill_set_elements) - filter = ColumnUInt8::create(block.rows()); + filter = ColumnUInt8::create(rows); switch (data.type) { @@ -224,16 +224,16 @@ bool Set::insertFromBlock(const Block & block) } -ColumnPtr Set::execute(const Block & block, bool negative) const +ColumnPtr Set::execute(const ColumnsWithTypeAndName & columns, bool negative) const { - size_t num_key_columns = block.columns(); + size_t num_key_columns = columns.size(); if (0 == num_key_columns) throw Exception("Logical error: no columns passed to Set::execute method.", ErrorCodes::LOGICAL_ERROR); auto res = ColumnUInt8::create(); ColumnUInt8::Container & vec_res = res->getData(); - vec_res.resize(block.safeGetByPosition(0).column->size()); + vec_res.resize(columns.at(0).column->size()); if (vec_res.empty()) return res; @@ -264,7 +264,7 @@ ColumnPtr Set::execute(const Block & block, bool negative) const { ColumnPtr result; - const auto & column_before_cast = block.safeGetByPosition(i); + const auto & column_before_cast = columns.at(i); ColumnWithTypeAndName column_to_cast = {column_before_cast.column->convertToFullColumnIfConst(), column_before_cast.type, column_before_cast.name}; diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h index 9bf6630b844..727a2c144a1 100644 --- a/src/Interpreters/Set.h +++ b/src/Interpreters/Set.h @@ -42,10 +42,10 @@ public: /** Create a Set from stream. * Call setHeader, then call insertFromBlock for each block. */ - void setHeader(const Block & header); + void setHeader(const ColumnsWithTypeAndName & header); /// Returns false, if some limit was exceeded and no need to insert more data. - bool insertFromBlock(const Block & block); + bool insertFromBlock(const ColumnsWithTypeAndName & columns); /// Call after all blocks were inserted. To get the information that set is already created. void finishInsert() { is_created = true; } @@ -54,7 +54,7 @@ public: /** For columns of 'block', check belonging of corresponding rows to the set. * Return UInt8 column with the result. */ - ColumnPtr execute(const Block & block, bool negative) const; + ColumnPtr execute(const ColumnsWithTypeAndName & columns, bool negative) const; bool empty() const; size_t getTotalRowCount() const; diff --git a/src/Processors/Transforms/CreatingSetsTransform.cpp b/src/Processors/Transforms/CreatingSetsTransform.cpp index 6f69765ee23..d9b383030d3 100644 --- a/src/Processors/Transforms/CreatingSetsTransform.cpp +++ b/src/Processors/Transforms/CreatingSetsTransform.cpp @@ -84,7 +84,7 @@ void CreatingSetsTransform::init() is_initialized = true; if (subquery.set) - subquery.set->setHeader(getInputPort().getHeader()); + subquery.set->setHeader(getInputPort().getHeader().getColumnsWithTypeAndName()); watch.restart(); startSubquery(); @@ -97,7 +97,7 @@ void CreatingSetsTransform::consume(Chunk chunk) if (!done_with_set) { - if (!subquery.set->insertFromBlock(block)) + if (!subquery.set->insertFromBlock(block.getColumnsWithTypeAndName())) done_with_set = true; } diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 67fd89f5098..c16b60af45f 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -146,13 +146,13 @@ StorageSet::StorageSet( Block header = getInMemoryMetadataPtr()->getSampleBlock(); header = header.sortColumns(); - set->setHeader(header); + set->setHeader(header.getColumnsWithTypeAndName()); restore(); } -void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block); } +void StorageSet::insertBlock(const Block & block) { set->insertFromBlock(block.getColumnsWithTypeAndName()); } void StorageSet::finishInsert() { set->finishInsert(); } size_t StorageSet::getSize() const { return set->getTotalRowCount(); } @@ -170,7 +170,7 @@ void StorageSet::truncate(const ASTPtr &, const StorageMetadataPtr & metadata_sn increment = 0; set = std::make_shared(SizeLimits(), false, true); - set->setHeader(header); + set->setHeader(header.getColumnsWithTypeAndName()); } diff --git a/src/Storages/System/StorageSystemZooKeeper.cpp b/src/Storages/System/StorageSystemZooKeeper.cpp index d19aef47616..cba10548852 100644 --- a/src/Storages/System/StorageSystemZooKeeper.cpp +++ b/src/Storages/System/StorageSystemZooKeeper.cpp @@ -97,12 +97,12 @@ static bool extractPathImpl(const IAST & elem, Paths & res, ContextPtr context) auto stream = interpreter_subquery->execute().getInputStream(); SizeLimits limites(context->getSettingsRef().max_rows_in_set, context->getSettingsRef().max_bytes_in_set, OverflowMode::THROW); Set set(limites, true, context->getSettingsRef().transform_null_in); - set.setHeader(stream->getHeader()); + set.setHeader(stream->getHeader().getColumnsWithTypeAndName()); stream->readPrefix(); while (Block block = stream->read()) { - set.insertFromBlock(block); + set.insertFromBlock(block.getColumnsWithTypeAndName()); } set.finishInsert(); stream->readSuffix(); From 53d7842877e6f5a77820540545aa0e7ebfbf3ba9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 18 Aug 2021 13:54:06 +0300 Subject: [PATCH 180/220] Update version_date.tsv after release 21.8.4.51 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 67238318e25..1bc21bfff17 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,3 +1,4 @@ +v21.8.4.51-lts 2021-08-17 v21.8.3.44-lts 2021-08-12 v21.7.7.47-stable 2021-08-09 v21.7.6.39-stable 2021-08-06 From 51ffc33457340a7fcb0a4e8d4d2d00952eeef997 Mon Sep 17 00:00:00 2001 From: Vasily Nemkov Date: Fri, 5 Mar 2021 16:57:16 +0200 Subject: [PATCH 181/220] Introduce sessions. This is required to add system.session_log table. --- programs/local/LocalServer.cpp | 9 +- programs/server/Server.cpp | 3 +- src/Core/MySQL/Authentication.cpp | 15 +- src/Core/MySQL/Authentication.h | 6 +- src/Core/PostgreSQLProtocol.h | 31 +- src/Interpreters/Context.cpp | 196 --------- src/Interpreters/Context.h | 34 +- src/Interpreters/Session.cpp | 392 ++++++++++++++++++ src/Interpreters/Session.h | 89 ++++ .../Formats/Impl/MySQLOutputFormat.cpp | 1 - src/Server/GRPCServer.cpp | 23 +- src/Server/HTTPHandler.cpp | 46 +- src/Server/HTTPHandler.h | 11 +- src/Server/MySQLHandler.cpp | 56 +-- src/Server/MySQLHandler.h | 4 +- src/Server/PostgreSQLHandler.cpp | 54 +-- src/Server/PostgreSQLHandler.h | 9 +- src/Server/TCPHandler.cpp | 69 +-- src/Server/TCPHandler.h | 5 +- src/TableFunctions/TableFunctionMySQL.cpp | 3 +- 20 files changed, 665 insertions(+), 391 deletions(-) create mode 100644 src/Interpreters/Session.cpp create mode 100644 src/Interpreters/Session.h diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index e256338a538..7f1bbe77d9c 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -376,13 +377,11 @@ void LocalServer::processQueries() /// we can't mutate global global_context (can lead to races, as it was already passed to some background threads) /// so we can't reuse it safely as a query context and need a copy here - auto context = Context::createCopy(global_context); + Session session(global_context, ClientInfo::Interface::TCP); + session.setUser("default", "", Poco::Net::SocketAddress{}); - context->makeSessionContext(); - context->makeQueryContext(); + auto context = session.makeQueryContext(""); - context->setUser("default", "", Poco::Net::SocketAddress{}); - context->setCurrentQueryId(""); applyCmdSettings(context); /// Use the same query_id (and thread group) for all queries diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 5520f920823..98c63f9896a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include #include @@ -1428,7 +1429,7 @@ if (ThreadFuzzer::instance().isEffective()) /// Must be done after initialization of `servers`, because async_metrics will access `servers` variable from its thread. async_metrics.start(); - global_context->enableNamedSessions(); + Session::enableNamedSessions(); { String level_str = config().getString("text_log.level", ""); diff --git a/src/Core/MySQL/Authentication.cpp b/src/Core/MySQL/Authentication.cpp index 658c86795b1..bc34b5637d6 100644 --- a/src/Core/MySQL/Authentication.cpp +++ b/src/Core/MySQL/Authentication.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -73,7 +74,7 @@ Native41::Native41(const String & password, const String & auth_plugin_data) } void Native41::authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, std::optional auth_response, Session & session, std::shared_ptr packet_endpoint, bool, const Poco::Net::SocketAddress & address) { if (!auth_response) @@ -86,7 +87,7 @@ void Native41::authenticate( if (auth_response->empty()) { - context->setUser(user_name, "", address); + session.setUser(user_name, "", address); return; } @@ -96,9 +97,9 @@ void Native41::authenticate( + " bytes, received: " + std::to_string(auth_response->size()) + " bytes.", ErrorCodes::UNKNOWN_EXCEPTION); - auto user = context->getAccessControlManager().read(user_name); + const auto user_authentication = session.getUserAuthentication(user_name); - Poco::SHA1Engine::Digest double_sha1_value = user->authentication.getPasswordDoubleSHA1(); + Poco::SHA1Engine::Digest double_sha1_value = user_authentication.getPasswordDoubleSHA1(); assert(double_sha1_value.size() == Poco::SHA1Engine::DIGEST_SIZE); Poco::SHA1Engine engine; @@ -111,7 +112,7 @@ void Native41::authenticate( { password_sha1[i] = digest[i] ^ static_cast((*auth_response)[i]); } - context->setUser(user_name, password_sha1, address); + session.setUser(user_name, password_sha1, address); } #if USE_SSL @@ -136,7 +137,7 @@ Sha256Password::Sha256Password(RSA & public_key_, RSA & private_key_, Poco::Logg } void Sha256Password::authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, std::optional auth_response, Session & session, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) { if (!auth_response) @@ -231,7 +232,7 @@ void Sha256Password::authenticate( password.pop_back(); } - context->setUser(user_name, password, address); + session.setUser(user_name, password, address); } #endif diff --git a/src/Core/MySQL/Authentication.h b/src/Core/MySQL/Authentication.h index acbda2bdb58..0dde8d10c0e 100644 --- a/src/Core/MySQL/Authentication.h +++ b/src/Core/MySQL/Authentication.h @@ -32,7 +32,7 @@ public: virtual String getAuthPluginData() = 0; virtual void authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, std::optional auth_response, Session & session, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) = 0; }; @@ -49,7 +49,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, std::optional auth_response, Session & session, std::shared_ptr packet_endpoint, bool /* is_secure_connection */, const Poco::Net::SocketAddress & address) override; private: @@ -69,7 +69,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, ContextMutablePtr context, + const String & user_name, std::optional auth_response, Session & session, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) override; private: diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 114abc0101f..19bcc727105 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -803,12 +804,12 @@ protected: static void setPassword( const String & user_name, const String & password, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { try { - context->setUser(user_name, password, address); + session.setUser(user_name, password, address); } catch (const Exception &) { @@ -822,7 +823,7 @@ protected: public: virtual void authenticate( const String & user_name, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) = 0; @@ -836,11 +837,11 @@ class NoPasswordAuth : public AuthenticationMethod public: void authenticate( const String & user_name, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { - setPassword(user_name, "", context, mt, address); + setPassword(user_name, "", session, mt, address); } Authentication::Type getType() const override @@ -854,7 +855,7 @@ class CleartextPasswordAuth : public AuthenticationMethod public: void authenticate( const String & user_name, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { @@ -864,7 +865,7 @@ public: if (type == Messaging::FrontMessageType::PASSWORD_MESSAGE) { std::unique_ptr password = mt.receive(); - setPassword(user_name, password->password, context, mt, address); + setPassword(user_name, password->password, session, mt, address); } else throw Exception( @@ -897,16 +898,24 @@ public: void authenticate( const String & user_name, - ContextMutablePtr context, + Session & session, Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { - auto user = context->getAccessControlManager().read(user_name); - Authentication::Type user_auth_type = user->authentication.getType(); + Authentication::Type user_auth_type; + try + { + user_auth_type = session.getUserAuthentication(user_name).getType(); + } + catch (const std::exception & e) + { + session.onLogInFailure(user_name, e); + throw; + } if (type_to_method.find(user_auth_type) != type_to_method.end()) { - type_to_method[user_auth_type]->authenticate(user_name, context, mt, address); + type_to_method[user_auth_type]->authenticate(user_name, session, mt, address); mt.send(Messaging::AuthenticationOk(), true); LOG_DEBUG(log, "Authentication for user {} was successful.", user_name); return; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index bd15af76db0..601127c99b5 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -100,7 +100,6 @@ namespace CurrentMetrics extern const Metric BackgroundMessageBrokerSchedulePoolTask; } - namespace DB { @@ -115,189 +114,11 @@ namespace ErrorCodes extern const int THERE_IS_NO_QUERY; extern const int NO_ELEMENTS_IN_CONFIG; extern const int TABLE_SIZE_EXCEEDS_MAX_DROP_SIZE_LIMIT; - extern const int SESSION_NOT_FOUND; - extern const int SESSION_IS_LOCKED; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; } -class NamedSessions -{ -public: - using Key = NamedSessionKey; - - ~NamedSessions() - { - try - { - { - std::lock_guard lock{mutex}; - quit = true; - } - - cond.notify_one(); - thread.join(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } - } - - /// Find existing session or create a new. - std::shared_ptr acquireSession( - const String & session_id, - ContextMutablePtr context, - std::chrono::steady_clock::duration timeout, - bool throw_if_not_found) - { - std::unique_lock lock(mutex); - - auto & user_name = context->client_info.current_user; - - if (user_name.empty()) - throw Exception("Empty user name.", ErrorCodes::LOGICAL_ERROR); - - Key key(user_name, session_id); - - auto it = sessions.find(key); - if (it == sessions.end()) - { - if (throw_if_not_found) - throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); - - /// Create a new session from current context. - it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; - } - else if (it->second->key.first != context->client_info.current_user) - { - throw Exception("Session belongs to a different user", ErrorCodes::SESSION_IS_LOCKED); - } - - /// Use existing session. - const auto & session = it->second; - - if (!session.unique()) - throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); - - session->context->client_info = context->client_info; - - return session; - } - - void releaseSession(NamedSession & session) - { - std::unique_lock lock(mutex); - scheduleCloseSession(session, lock); - } - -private: - class SessionKeyHash - { - public: - size_t operator()(const Key & key) const - { - SipHash hash; - hash.update(key.first); - hash.update(key.second); - return hash.get64(); - } - }; - - /// TODO it's very complicated. Make simple std::map with time_t or boost::multi_index. - using Container = std::unordered_map, SessionKeyHash>; - using CloseTimes = std::deque>; - Container sessions; - CloseTimes close_times; - std::chrono::steady_clock::duration close_interval = std::chrono::seconds(1); - std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); - UInt64 close_cycle = 0; - - void scheduleCloseSession(NamedSession & session, std::unique_lock &) - { - /// Push it on a queue of sessions to close, on a position corresponding to the timeout. - /// (timeout is measured from current moment of time) - - const UInt64 close_index = session.timeout / close_interval + 1; - const auto new_close_cycle = close_cycle + close_index; - - if (session.close_cycle != new_close_cycle) - { - session.close_cycle = new_close_cycle; - if (close_times.size() < close_index + 1) - close_times.resize(close_index + 1); - close_times[close_index].emplace_back(session.key); - } - } - - void cleanThread() - { - setThreadName("SessionCleaner"); - std::unique_lock lock{mutex}; - - while (true) - { - auto interval = closeSessions(lock); - - if (cond.wait_for(lock, interval, [this]() -> bool { return quit; })) - break; - } - } - - /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. - std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) - { - const auto now = std::chrono::steady_clock::now(); - - /// The time to close the next session did not come - if (now < close_cycle_time) - return close_cycle_time - now; /// Will sleep until it comes. - - const auto current_cycle = close_cycle; - - ++close_cycle; - close_cycle_time = now + close_interval; - - if (close_times.empty()) - return close_interval; - - auto & sessions_to_close = close_times.front(); - - for (const auto & key : sessions_to_close) - { - const auto session = sessions.find(key); - - if (session != sessions.end() && session->second->close_cycle <= current_cycle) - { - if (!session->second.unique()) - { - /// Skip but move it to close on the next cycle. - session->second->timeout = std::chrono::steady_clock::duration{0}; - scheduleCloseSession(*session->second, lock); - } - else - sessions.erase(session); - } - } - - close_times.pop_front(); - return close_interval; - } - - std::mutex mutex; - std::condition_variable cond; - std::atomic quit{false}; - ThreadFromGlobalPool thread{&NamedSessions::cleanThread, this}; -}; - - -void NamedSession::release() -{ - parent.releaseSession(*this); -} - - /** Set of known objects (environment), that could be used in query. * Shared (global) part. Order of members (especially, order of destruction) is very important. */ @@ -399,7 +220,6 @@ struct ContextSharedPart RemoteHostFilter remote_host_filter; /// Allowed URL from config.xml std::optional trace_collector; /// Thread collecting traces from threads executing queries - std::optional named_sessions; /// Controls named HTTP sessions. /// Clusters for distributed tables /// Initialized on demand (on distributed storages initialization) since Settings should be initialized @@ -587,7 +407,6 @@ void Context::copyFrom(const ContextPtr & other) Context::~Context() = default; - InterserverIOHandler & Context::getInterserverIOHandler() { return shared->interserver_io_handler; } std::unique_lock Context::getLock() const @@ -604,21 +423,6 @@ const MergeList & Context::getMergeList() const { return shared->merge_list; } ReplicatedFetchList & Context::getReplicatedFetchList() { return shared->replicated_fetch_list; } const ReplicatedFetchList & Context::getReplicatedFetchList() const { return shared->replicated_fetch_list; } - -void Context::enableNamedSessions() -{ - shared->named_sessions.emplace(); -} - -std::shared_ptr -Context::acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) -{ - if (!shared->named_sessions) - throw Exception("Support for named sessions is not enabled", ErrorCodes::NOT_IMPLEMENTED); - - return shared->named_sessions->acquireSession(session_id, shared_from_this(), timeout, session_check); -} - String Context::resolveDatabase(const String & database_name) const { String res = database_name.empty() ? getCurrentDatabase() : database_name; diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index d3a77e0039b..0bb32bb7b43 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -107,6 +107,7 @@ using StoragePolicySelectorPtr = std::shared_ptr; struct PartUUIDs; using PartUUIDsPtr = std::shared_ptr; class KeeperStorageDispatcher; +class Session; class IOutputFormat; using OutputFormatPtr = std::shared_ptr; @@ -287,8 +288,6 @@ public: OpenTelemetryTraceContext query_trace_context; private: - friend class NamedSessions; - using SampleBlockCache = std::unordered_map; mutable SampleBlockCache sample_block_cache; @@ -591,10 +590,6 @@ public: std::optional getTCPPortSecure() const; - /// Allow to use named sessions. The thread will be run to cleanup sessions after timeout has expired. - /// The method must be called at the server startup. - void enableNamedSessions(); - std::shared_ptr acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); /// For methods below you may need to acquire the context lock by yourself. @@ -607,6 +602,7 @@ public: bool hasSessionContext() const { return !session_context.expired(); } ContextMutablePtr getGlobalContext() const; + bool hasGlobalContext() const { return !global_context.expired(); } bool isGlobalContext() const { @@ -852,32 +848,6 @@ private: StoragePolicySelectorPtr getStoragePolicySelector(std::lock_guard & lock) const; DiskSelectorPtr getDiskSelector(std::lock_guard & /* lock */) const; - - /// If the password is not set, the password will not be checked - void setUserImpl(const String & name, const std::optional & password, const Poco::Net::SocketAddress & address); -}; - - -class NamedSessions; - -/// User name and session identifier. Named sessions are local to users. -using NamedSessionKey = std::pair; - -/// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. -struct NamedSession -{ - NamedSessionKey key; - UInt64 close_cycle = 0; - ContextMutablePtr context; - std::chrono::steady_clock::duration timeout; - NamedSessions & parent; - - NamedSession(NamedSessionKey key_, ContextPtr context_, std::chrono::steady_clock::duration timeout_, NamedSessions & parent_) - : key(key_), context(Context::createCopy(context_)), timeout(timeout_), parent(parent_) - { - } - - void release(); }; } diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp new file mode 100644 index 00000000000..acebc182a64 --- /dev/null +++ b/src/Interpreters/Session.cpp @@ -0,0 +1,392 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int SESSION_NOT_FOUND; + extern const int SESSION_IS_LOCKED; + extern const int NOT_IMPLEMENTED; +} + +class NamedSessionsStorage; + +/// User name and session identifier. Named sessions are local to users. +using NamedSessionKey = std::pair; + +/// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. +struct NamedSessionData +{ + NamedSessionKey key; + UInt64 close_cycle = 0; + ContextMutablePtr context; + std::chrono::steady_clock::duration timeout; + NamedSessionsStorage & parent; + + NamedSessionData(NamedSessionKey key_, ContextPtr context_, std::chrono::steady_clock::duration timeout_, NamedSessionsStorage & parent_) + : key(std::move(key_)), context(Context::createCopy(context_)), timeout(timeout_), parent(parent_) + {} + + void release(); +}; + +class NamedSessionsStorage +{ +public: + using Key = NamedSessionKey; + + ~NamedSessionsStorage() + { + try + { + { + std::lock_guard lock{mutex}; + quit = true; + } + + cond.notify_one(); + thread.join(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + /// Find existing session or create a new. + std::shared_ptr acquireSession( + const String & session_id, + ContextMutablePtr context, + std::chrono::steady_clock::duration timeout, + bool throw_if_not_found) + { + std::unique_lock lock(mutex); + + const auto & client_info = context->getClientInfo(); + const auto & user_name = client_info.current_user; + + if (user_name.empty()) + throw Exception("Empty user name.", ErrorCodes::LOGICAL_ERROR); + + Key key(user_name, session_id); + + auto it = sessions.find(key); + if (it == sessions.end()) + { + if (throw_if_not_found) + throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); + + /// Create a new session from current context. + it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; + } + else if (it->second->key.first != client_info.current_user) + { + throw Exception("Session belongs to a different user", ErrorCodes::SESSION_IS_LOCKED); + } + + /// Use existing session. + const auto & session = it->second; + + if (!session.unique()) + throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); + + session->context->getClientInfo() = client_info; + + return session; + } + + void releaseSession(NamedSessionData & session) + { + std::unique_lock lock(mutex); + scheduleCloseSession(session, lock); + } + +private: + class SessionKeyHash + { + public: + size_t operator()(const Key & key) const + { + SipHash hash; + hash.update(key.first); + hash.update(key.second); + return hash.get64(); + } + }; + + /// TODO it's very complicated. Make simple std::map with time_t or boost::multi_index. + using Container = std::unordered_map, SessionKeyHash>; + using CloseTimes = std::deque>; + Container sessions; + CloseTimes close_times; + std::chrono::steady_clock::duration close_interval = std::chrono::seconds(1); + std::chrono::steady_clock::time_point close_cycle_time = std::chrono::steady_clock::now(); + UInt64 close_cycle = 0; + + void scheduleCloseSession(NamedSessionData & session, std::unique_lock &) + { + /// Push it on a queue of sessions to close, on a position corresponding to the timeout. + /// (timeout is measured from current moment of time) + + const UInt64 close_index = session.timeout / close_interval + 1; + const auto new_close_cycle = close_cycle + close_index; + + if (session.close_cycle != new_close_cycle) + { + session.close_cycle = new_close_cycle; + if (close_times.size() < close_index + 1) + close_times.resize(close_index + 1); + close_times[close_index].emplace_back(session.key); + } + } + + void cleanThread() + { + setThreadName("SessionCleaner"); + std::unique_lock lock{mutex}; + + while (true) + { + auto interval = closeSessions(lock); + + if (cond.wait_for(lock, interval, [this]() -> bool { return quit; })) + break; + } + } + + /// Close sessions, that has been expired. Returns how long to wait for next session to be expired, if no new sessions will be added. + std::chrono::steady_clock::duration closeSessions(std::unique_lock & lock) + { + const auto now = std::chrono::steady_clock::now(); + + /// The time to close the next session did not come + if (now < close_cycle_time) + return close_cycle_time - now; /// Will sleep until it comes. + + const auto current_cycle = close_cycle; + + ++close_cycle; + close_cycle_time = now + close_interval; + + if (close_times.empty()) + return close_interval; + + auto & sessions_to_close = close_times.front(); + + for (const auto & key : sessions_to_close) + { + const auto session = sessions.find(key); + + if (session != sessions.end() && session->second->close_cycle <= current_cycle) + { + if (!session->second.unique()) + { + /// Skip but move it to close on the next cycle. + session->second->timeout = std::chrono::steady_clock::duration{0}; + scheduleCloseSession(*session->second, lock); + } + else + sessions.erase(session); + } + } + + close_times.pop_front(); + return close_interval; + } + + std::mutex mutex; + std::condition_variable cond; + std::atomic quit{false}; + ThreadFromGlobalPool thread{&NamedSessionsStorage::cleanThread, this}; +}; + + +void NamedSessionData::release() +{ + parent.releaseSession(*this); +} + +std::optional Session::named_sessions = std::nullopt; + +void Session::enableNamedSessions() +{ + named_sessions.emplace(); +} + +Session::Session(const ContextPtr & context_to_copy, ClientInfo::Interface interface, std::optional default_format) + : session_context(Context::createCopy(context_to_copy)), + initial_session_context(session_context) +{ + session_context->makeSessionContext(); + session_context->getClientInfo().interface = interface; + + if (default_format) + session_context->setDefaultFormat(*default_format); +} + +Session::Session(Session &&) = default; + +Session::~Session() +{ + releaseNamedSession(); + + if (access) + { + auto user = access->getUser(); + if (user) + onLogOut(); + } +} + +Authentication Session::getUserAuthentication(const String & user_name) const +{ + return session_context->getAccessControlManager().read(user_name)->authentication; +} + +void Session::setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address) +{ + try + { + session_context->setUser(credentials, address); + + // Caching access just in case if context is going to be replaced later (e.g. with context of NamedSessionData) + access = session_context->getAccess(); + + // Check if this is a not an intercluster session, but the real one. + if (access && access->getUser() && dynamic_cast(&credentials)) + { + onLogInSuccess(); + } + } + catch (const std::exception & e) + { + onLogInFailure(credentials.getUserName(), e); + throw; + } +} + +void Session::setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address) +{ + setUser(BasicCredentials(name, password), address); +} + +void Session::onLogInSuccess() +{ +} + +void Session::onLogInFailure(const String & /* user_name */, const std::exception & /* failure_reason */) +{ +} + +void Session::onLogOut() +{ +} + +void Session::promoteToNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) +{ + if (!named_sessions) + throw Exception("Support for named sessions is not enabled", ErrorCodes::NOT_IMPLEMENTED); + + auto new_named_session = named_sessions->acquireSession(session_id, session_context, timeout, session_check); + + // Must retain previous client info cause otherwise source client address and port, + // and other stuff are reused from previous user of the said session. + const ClientInfo prev_client_info = session_context->getClientInfo(); + + session_context = new_named_session->context; + session_context->getClientInfo() = prev_client_info; + session_context->makeSessionContext(); + + named_session.swap(new_named_session); +} + +/// Early release a NamedSessionData. +void Session::releaseNamedSession() +{ + if (named_session) + { + named_session->release(); + named_session.reset(); + } + + session_context = initial_session_context; +} + +ContextMutablePtr Session::makeQueryContext(const String & query_id) const +{ + ContextMutablePtr new_query_context = Context::createCopy(session_context); + + new_query_context->setCurrentQueryId(query_id); + new_query_context->setSessionContext(session_context); + new_query_context->makeQueryContext(); + + ClientInfo & client_info = new_query_context->getClientInfo(); + client_info.initial_user = client_info.current_user; + client_info.initial_query_id = client_info.current_query_id; + client_info.initial_address = client_info.current_address; + + return new_query_context; +} + +ContextPtr Session::sessionContext() const +{ + return session_context; +} + +ContextMutablePtr Session::mutableSessionContext() +{ + return session_context; +} + +ClientInfo & Session::getClientInfo() +{ + return session_context->getClientInfo(); +} + +const ClientInfo & Session::getClientInfo() const +{ + return session_context->getClientInfo(); +} + +const Settings & Session::getSettings() const +{ + return session_context->getSettingsRef(); +} + +void Session::setQuotaKey(const String & quota_key) +{ + session_context->setQuotaKey(quota_key); +} + +String Session::getCurrentDatabase() const +{ + return session_context->getCurrentDatabase(); +} + +void Session::setCurrentDatabase(const String & name) +{ + session_context->setCurrentDatabase(name); +} + +} diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h new file mode 100644 index 00000000000..300ed779c49 --- /dev/null +++ b/src/Interpreters/Session.h @@ -0,0 +1,89 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include + +namespace Poco::Net { class SocketAddress; } + +namespace DB +{ +class Credentials; +class ContextAccess; +struct Settings; +class Authentication; +struct NamedSessionData; +class NamedSessionsStorage; + +/** Represents user-session from the server perspective, + * basically it is just a smaller subset of Context API, simplifies Context management. + * + * Holds session context, facilitates acquisition of NamedSession and proper creation of query contexts. + * Adds log in, log out and login failure events to the SessionLog. + */ +class Session +{ + static std::optional named_sessions; + +public: + /// Allow to use named sessions. The thread will be run to cleanup sessions after timeout has expired. + /// The method must be called at the server startup. + static void enableNamedSessions(); + +// static Session makeSessionFromCopyOfContext(const ContextPtr & _context_to_copy); + Session(const ContextPtr & context_to_copy, ClientInfo::Interface interface, std::optional default_format = std::nullopt); + virtual ~Session(); + + Session(const Session &) = delete; + Session& operator=(const Session &) = delete; + + Session(Session &&); +// Session& operator=(Session &&); + + Authentication getUserAuthentication(const String & user_name) const; + void setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address); + void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address); + + /// Handle login and logout events. + void onLogInSuccess(); + void onLogInFailure(const String & user_name, const std::exception & /* failure_reason */); + void onLogOut(); + + /** Propmotes current session to a named session. + * + * that is: re-uses or creates NamedSession and then piggybacks on it's context, + * retaining ClientInfo of current session_context. + * Acquired named_session is then released in the destructor. + */ + void promoteToNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); + /// Early release a NamedSession. + void releaseNamedSession(); + + ContextMutablePtr makeQueryContext(const String & query_id) const; + + ContextPtr sessionContext() const; + ContextMutablePtr mutableSessionContext(); + + ClientInfo & getClientInfo(); + const ClientInfo & getClientInfo() const; + + const Settings & getSettings() const; + + void setQuotaKey(const String & quota_key); + + String getCurrentDatabase() const; + void setCurrentDatabase(const String & name); + +private: + ContextMutablePtr session_context; + // So that Session can be used after forced release of named_session. + const ContextMutablePtr initial_session_context; + std::shared_ptr access; + std::shared_ptr named_session; +}; + +} diff --git a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp index 6fdcc544a18..4c6f4624ad4 100644 --- a/src/Processors/Formats/Impl/MySQLOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLOutputFormat.cpp @@ -64,7 +64,6 @@ void MySQLOutputFormat::initialize() } } - void MySQLOutputFormat::consume(Chunk chunk) { initialize(); diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index b90b0c33f17..f03d0ae5f9f 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -560,7 +561,7 @@ namespace IServer & iserver; Poco::Logger * log = nullptr; - std::shared_ptr session; + std::shared_ptr session; ContextMutablePtr query_context; std::optional query_scope; String query_text; @@ -690,30 +691,28 @@ namespace } /// Create context. - query_context = Context::createCopy(iserver.context()); - + session = std::make_shared(iserver.context(), ClientInfo::Interface::GRPC); /// Authentication. - query_context->setUser(user, password, user_address); - query_context->setCurrentQueryId(query_info.query_id()); + session->setUser(user, password, user_address); if (!quota_key.empty()) - query_context->setQuotaKey(quota_key); + session->setQuotaKey(quota_key); /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. if (!query_info.session_id().empty()) { - session = query_context->acquireNamedSession( - query_info.session_id(), getSessionTimeout(query_info, iserver.config()), query_info.session_check()); - query_context = Context::createCopy(session->context); - query_context->setSessionContext(session->context); + session->promoteToNamedSession( + query_info.session_id(), + getSessionTimeout(query_info, iserver.config()), + query_info.session_check()); } + query_context = session->makeQueryContext(query_info.query_id()); query_scope.emplace(query_context); /// Set client info. ClientInfo & client_info = query_context->getClientInfo(); client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.interface = ClientInfo::Interface::GRPC; client_info.initial_user = client_info.current_user; client_info.initial_query_id = client_info.current_query_id; client_info.initial_address = client_info.current_address; @@ -1254,8 +1253,6 @@ namespace io = {}; query_scope.reset(); query_context.reset(); - if (session) - session->release(); session.reset(); } diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 7c9ff0637f0..0e6b7d57b7c 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -275,7 +276,6 @@ HTTPHandler::~HTTPHandler() bool HTTPHandler::authenticateUser( - ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response) @@ -352,7 +352,7 @@ bool HTTPHandler::authenticateUser( else { if (!request_credentials) - request_credentials = request_context->makeGSSAcceptorContext(); + request_credentials = request_session->sessionContext()->makeGSSAcceptorContext(); auto * gss_acceptor_context = dynamic_cast(request_credentials.get()); if (!gss_acceptor_context) @@ -379,9 +379,8 @@ bool HTTPHandler::authenticateUser( /// Set client info. It will be used for quota accounting parameters in 'setUser' method. - ClientInfo & client_info = context->getClientInfo(); + ClientInfo & client_info = request_session->getClientInfo(); client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.interface = ClientInfo::Interface::HTTP; ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN; if (request.getMethod() == HTTPServerRequest::HTTP_GET) @@ -396,7 +395,7 @@ bool HTTPHandler::authenticateUser( try { - context->setUser(*request_credentials, request.clientAddress()); + request_session->setUser(*request_credentials, request.clientAddress()); } catch (const Authentication::Require & required_credentials) { @@ -413,7 +412,7 @@ bool HTTPHandler::authenticateUser( } catch (const Authentication::Require & required_credentials) { - request_credentials = request_context->makeGSSAcceptorContext(); + request_credentials = request_session->sessionContext()->makeGSSAcceptorContext(); if (required_credentials.getRealm().empty()) response.set("WWW-Authenticate", "Negotiate"); @@ -428,7 +427,7 @@ bool HTTPHandler::authenticateUser( request_credentials.reset(); if (!quota_key.empty()) - context->setQuotaKey(quota_key); + request_session->setQuotaKey(quota_key); /// Query sent through HTTP interface is initial. client_info.initial_user = client_info.current_user; @@ -439,7 +438,6 @@ bool HTTPHandler::authenticateUser( void HTTPHandler::processQuery( - ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response, @@ -450,13 +448,11 @@ void HTTPHandler::processQuery( LOG_TRACE(log, "Request URI: {}", request.getURI()); - if (!authenticateUser(context, request, params, response)) + if (!authenticateUser(request, params, response)) return; // '401 Unauthorized' response with 'Negotiate' has been sent at this point. /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. - - std::shared_ptr session; String session_id; std::chrono::steady_clock::duration session_timeout; bool session_is_set = params.has("session_id"); @@ -467,16 +463,11 @@ void HTTPHandler::processQuery( session_id = params.get("session_id"); session_timeout = parseSessionTimeout(config, params); std::string session_check = params.get("session_check", ""); - - session = context->acquireNamedSession(session_id, session_timeout, session_check == "1"); - - context->copyFrom(session->context); /// FIXME: maybe move this part to HandleRequest(), copyFrom() is used only here. - context->setSessionContext(session->context); + request_session->promoteToNamedSession(session_id, session_timeout, session_check == "1"); } SCOPE_EXIT({ - if (session) - session->release(); + request_session->releaseNamedSession(); }); // Parse the OpenTelemetry traceparent header. @@ -485,9 +476,10 @@ void HTTPHandler::processQuery( #if !defined(ARCADIA_BUILD) if (request.has("traceparent")) { + ClientInfo & client_info = request_session->getClientInfo(); std::string opentelemetry_traceparent = request.get("traceparent"); std::string error; - if (!context->getClientInfo().client_trace_context.parseTraceparentHeader( + if (!client_info.client_trace_context.parseTraceparentHeader( opentelemetry_traceparent, error)) { throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, @@ -495,12 +487,12 @@ void HTTPHandler::processQuery( opentelemetry_traceparent, error); } - context->getClientInfo().client_trace_context.tracestate = request.get("tracestate", ""); + client_info.client_trace_context.tracestate = request.get("tracestate", ""); } #endif // Set the query id supplied by the user, if any, and also update the OpenTelemetry fields. - context->setCurrentQueryId(params.get("query_id", request.get("X-ClickHouse-Query-Id", ""))); + auto context = request_session->makeQueryContext(params.get("query_id", request.get("X-ClickHouse-Query-Id", ""))); ClientInfo & client_info = context->getClientInfo(); client_info.initial_query_id = client_info.current_query_id; @@ -866,16 +858,16 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse SCOPE_EXIT({ // If there is no request_credentials instance waiting for the next round, then the request is processed, - // so no need to preserve request_context either. + // so no need to preserve request_session either. // Needs to be performed with respect to the other destructors in the scope though. if (!request_credentials) - request_context.reset(); + request_session.reset(); }); - if (!request_context) + if (!request_session) { // Context should be initialized before anything, for correct memory accounting. - request_context = Context::createCopy(server.context()); + request_session = std::make_shared(server.context(), ClientInfo::Interface::HTTP); request_credentials.reset(); } @@ -894,7 +886,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse if (request.getVersion() == HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); - HTMLForm params(request_context->getSettingsRef(), request); + HTMLForm params(request_session->getSettings(), request); with_stacktrace = params.getParsed("stacktrace", false); /// FIXME: maybe this check is already unnecessary. @@ -906,7 +898,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse ErrorCodes::HTTP_LENGTH_REQUIRED); } - processQuery(request_context, request, params, response, used_output, query_scope); + processQuery(request, params, response, used_output, query_scope); LOG_DEBUG(log, (request_credentials ? "Authentication in progress..." : "Done processing query")); } catch (...) diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index 2149a7ca55c..bca73ca7cb8 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -18,6 +18,7 @@ namespace Poco { class Logger; } namespace DB { +class Session; class Credentials; class IServer; class WriteBufferFromHTTPServerResponse; @@ -71,25 +72,23 @@ private: CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; - // The request_context and the request_credentials instances may outlive a single request/response loop. + // The request_session and the request_credentials instances may outlive a single request/response loop. // This happens only when the authentication mechanism requires more than a single request/response exchange (e.g., SPNEGO). - ContextMutablePtr request_context; + std::shared_ptr request_session; std::unique_ptr request_credentials; // Returns true when the user successfully authenticated, - // the request_context instance will be configured accordingly, and the request_credentials instance will be dropped. + // the request_session instance will be configured accordingly, and the request_credentials instance will be dropped. // Returns false when the user is not authenticated yet, and the 'Negotiate' response is sent, - // the request_context and request_credentials instances are preserved. + // the request_session and request_credentials instances are preserved. // Throws an exception if authentication failed. bool authenticateUser( - ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response); /// Also initializes 'used_output'. void processQuery( - ContextMutablePtr context, HTTPServerRequest & request, HTMLForm & params, HTTPServerResponse & response, diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index 52182257ac9..f2ac1184640 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -70,7 +71,6 @@ MySQLHandler::MySQLHandler(IServer & server_, const Poco::Net::StreamSocket & so , server(server_) , log(&Poco::Logger::get("MySQLHandler")) , connection_id(connection_id_) - , connection_context(Context::createCopy(server.context())) , auth_plugin(new MySQLProtocol::Authentication::Native41()) { server_capabilities = CLIENT_PROTOCOL_41 | CLIENT_SECURE_CONNECTION | CLIENT_PLUGIN_AUTH | CLIENT_PLUGIN_AUTH_LENENC_CLIENT_DATA | CLIENT_CONNECT_WITH_DB | CLIENT_DEPRECATE_EOF; @@ -87,11 +87,13 @@ void MySQLHandler::run() { setThreadName("MySQLHandler"); ThreadStatus thread_status; - connection_context->makeSessionContext(); - connection_context->getClientInfo().interface = ClientInfo::Interface::MYSQL; - connection_context->setDefaultFormat("MySQLWire"); - connection_context->getClientInfo().connection_id = connection_id; - connection_context->getClientInfo().query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + + session = std::make_shared(server.context(), ClientInfo::Interface::MYSQL, "MySQLWire"); + auto & session_client_info = session->getClientInfo(); + + session_client_info.current_address = socket().peerAddress(); + session_client_info.connection_id = connection_id; + session_client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; in = std::make_shared(socket()); out = std::make_shared(socket()); @@ -125,14 +127,12 @@ void MySQLHandler::run() authenticate(handshake_response.username, handshake_response.auth_plugin_name, handshake_response.auth_response); - connection_context->getClientInfo().initial_user = handshake_response.username; + session_client_info.initial_user = handshake_response.username; try { if (!handshake_response.database.empty()) - connection_context->setCurrentDatabase(handshake_response.database); - connection_context->setCurrentQueryId(Poco::format("mysql:%lu", connection_id)); - + session->setCurrentDatabase(handshake_response.database); } catch (const Exception & exc) { @@ -246,25 +246,26 @@ void MySQLHandler::finishHandshake(MySQLProtocol::ConnectionPhase::HandshakeResp void MySQLHandler::authenticate(const String & user_name, const String & auth_plugin_name, const String & initial_auth_response) { + // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used. + DB::Authentication::Type user_auth_type; try { - // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used. - auto user = connection_context->getAccessControlManager().read(user_name); - const DB::Authentication::Type user_auth_type = user->authentication.getType(); - if (user_auth_type == DB::Authentication::SHA256_PASSWORD) - { - authPluginSSL(); - } - - std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; - auth_plugin->authenticate(user_name, auth_response, connection_context, packet_endpoint, secure_connection, socket().peerAddress()); + user_auth_type = session->getUserAuthentication(user_name).getType(); } - catch (const Exception & exc) + catch (const std::exception & e) { - LOG_ERROR(log, "Authentication for user {} failed.", user_name); - packet_endpoint->sendPacket(ERRPacket(exc.code(), "00000", exc.message()), true); + session->onLogInFailure(user_name, e); throw; } + + if (user_auth_type == DB::Authentication::SHA256_PASSWORD) + { + authPluginSSL(); + } + + std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; + auth_plugin->authenticate(user_name, auth_response, *session, packet_endpoint, secure_connection, socket().peerAddress()); + LOG_DEBUG(log, "Authentication for user {} succeeded.", user_name); } @@ -273,7 +274,7 @@ void MySQLHandler::comInitDB(ReadBuffer & payload) String database; readStringUntilEOF(database, payload); LOG_DEBUG(log, "Setting current database to {}", database); - connection_context->setCurrentDatabase(database); + session->setCurrentDatabase(database); packet_endpoint->sendPacket(OKPacket(0, client_capabilities, 0, 0, 1), true); } @@ -281,8 +282,9 @@ void MySQLHandler::comFieldList(ReadBuffer & payload) { ComFieldList packet; packet.readPayloadWithUnpacked(payload); - String database = connection_context->getCurrentDatabase(); - StoragePtr table_ptr = DatabaseCatalog::instance().getTable({database, packet.table}, connection_context); + const auto session_context = session->sessionContext(); + String database = session_context->getCurrentDatabase(); + StoragePtr table_ptr = DatabaseCatalog::instance().getTable({database, packet.table}, session_context); auto metadata_snapshot = table_ptr->getInMemoryMetadataPtr(); for (const NameAndTypePair & column : metadata_snapshot->getColumns().getAll()) { @@ -329,7 +331,7 @@ void MySQLHandler::comQuery(ReadBuffer & payload) ReadBufferFromString replacement(replacement_query); - auto query_context = Context::createCopy(connection_context); + auto query_context = session->makeQueryContext(Poco::format("mysql:%lu", connection_id)); std::atomic affected_rows {0}; auto prev = query_context->getProgressCallback(); diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index 96467797105..c57cb7d8f65 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -17,6 +17,8 @@ # include #endif +#include + namespace CurrentMetrics { extern const Metric MySQLConnection; @@ -61,7 +63,7 @@ protected: uint8_t sequence_id = 0; MySQLProtocol::PacketEndpointPtr packet_endpoint; - ContextMutablePtr connection_context; + std::shared_ptr session; using ReplacementFn = std::function; using Replacements = std::unordered_map; diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 1e98ed2e134..ae21d387e73 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "PostgreSQLHandler.h" #include #include @@ -33,7 +34,6 @@ PostgreSQLHandler::PostgreSQLHandler( std::vector> & auth_methods_) : Poco::Net::TCPServerConnection(socket_) , server(server_) - , connection_context(Context::createCopy(server.context())) , ssl_enabled(ssl_enabled_) , connection_id(connection_id_) , authentication_manager(auth_methods_) @@ -52,14 +52,15 @@ void PostgreSQLHandler::run() { setThreadName("PostgresHandler"); ThreadStatus thread_status; - connection_context->makeSessionContext(); - connection_context->getClientInfo().interface = ClientInfo::Interface::POSTGRESQL; - connection_context->setDefaultFormat("PostgreSQLWire"); - connection_context->getClientInfo().query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + + Session session(server.context(), ClientInfo::Interface::POSTGRESQL, "PostgreSQLWire"); + auto & session_client_info = session.getClientInfo(); + + session_client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; try { - if (!startup()) + if (!startup(session)) return; while (true) @@ -70,7 +71,7 @@ void PostgreSQLHandler::run() switch (message_type) { case PostgreSQLProtocol::Messaging::FrontMessageType::QUERY: - processQuery(); + processQuery(session); break; case PostgreSQLProtocol::Messaging::FrontMessageType::TERMINATE: LOG_DEBUG(log, "Client closed the connection"); @@ -109,7 +110,7 @@ void PostgreSQLHandler::run() } -bool PostgreSQLHandler::startup() +bool PostgreSQLHandler::startup(Session & session) { Int32 payload_size; Int32 info; @@ -118,23 +119,17 @@ bool PostgreSQLHandler::startup() if (static_cast(info) == PostgreSQLProtocol::Messaging::FrontMessageType::CANCEL_REQUEST) { LOG_DEBUG(log, "Client issued request canceling"); - cancelRequest(); + cancelRequest(session); return false; } std::unique_ptr start_up_msg = receiveStartupMessage(payload_size); - authentication_manager.authenticate(start_up_msg->user, connection_context, *message_transport, socket().peerAddress()); - - std::random_device rd; - std::mt19937 gen(rd()); - std::uniform_int_distribution dis(0, INT32_MAX); - secret_key = dis(gen); + authentication_manager.authenticate(start_up_msg->user, session, *message_transport, socket().peerAddress()); try { if (!start_up_msg->database.empty()) - connection_context->setCurrentDatabase(start_up_msg->database); - connection_context->setCurrentQueryId(Poco::format("postgres:%d:%d", connection_id, secret_key)); + session.setCurrentDatabase(start_up_msg->database); } catch (const Exception & exc) { @@ -212,10 +207,11 @@ void PostgreSQLHandler::sendParameterStatusData(PostgreSQLProtocol::Messaging::S message_transport->flush(); } -void PostgreSQLHandler::cancelRequest() +void PostgreSQLHandler::cancelRequest(Session & session) { - connection_context->setCurrentQueryId(""); - connection_context->setDefaultFormat("Null"); + // TODO (nemkov): maybe run cancellation query with session context? + auto query_context = session.makeQueryContext(std::string{}); + query_context->setDefaultFormat("Null"); std::unique_ptr msg = message_transport->receiveWithPayloadSize(8); @@ -223,7 +219,7 @@ void PostgreSQLHandler::cancelRequest() String query = Poco::format("KILL QUERY WHERE query_id = 'postgres:%d:%d'", msg->process_id, msg->secret_key); ReadBufferFromString replacement(query); - executeQuery(replacement, *out, true, connection_context, {}); + executeQuery(replacement, *out, true, query_context, {}); } inline std::unique_ptr PostgreSQLHandler::receiveStartupMessage(int payload_size) @@ -246,7 +242,7 @@ inline std::unique_ptr PostgreSQL return message; } -void PostgreSQLHandler::processQuery() +void PostgreSQLHandler::processQuery(Session & session) { try { @@ -269,18 +265,24 @@ void PostgreSQLHandler::processQuery() return; } - const auto & settings = connection_context->getSettingsRef(); + const auto & settings = session.getSettings(); std::vector queries; auto parse_res = splitMultipartQuery(query->query, queries, settings.max_query_size, settings.max_parser_depth); if (!parse_res.second) throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR); + std::random_device rd; + std::mt19937 gen(rd()); + std::uniform_int_distribution dis(0, INT32_MAX); + for (const auto & spl_query : queries) { - /// FIXME why do we execute all queries in a single connection context? - CurrentThread::QueryScope query_scope{connection_context}; + secret_key = dis(gen); + auto query_context = session.makeQueryContext(Poco::format("postgres:%d:%d", connection_id, secret_key)); + + CurrentThread::QueryScope query_scope{query_context}; ReadBufferFromString read_buf(spl_query); - executeQuery(read_buf, *out, false, connection_context, {}); + executeQuery(read_buf, *out, false, query_context, {}); PostgreSQLProtocol::Messaging::CommandComplete::Command command = PostgreSQLProtocol::Messaging::CommandComplete::classifyQuery(spl_query); diff --git a/src/Server/PostgreSQLHandler.h b/src/Server/PostgreSQLHandler.h index 9aaad1d7aa7..cf4a6620063 100644 --- a/src/Server/PostgreSQLHandler.h +++ b/src/Server/PostgreSQLHandler.h @@ -18,6 +18,8 @@ namespace CurrentMetrics namespace DB { +class Session; + /** PostgreSQL wire protocol implementation. * For more info see https://www.postgresql.org/docs/current/protocol.html */ @@ -37,7 +39,6 @@ private: Poco::Logger * log = &Poco::Logger::get("PostgreSQLHandler"); IServer & server; - ContextMutablePtr connection_context; bool ssl_enabled = false; Int32 connection_id = 0; Int32 secret_key = 0; @@ -56,7 +57,7 @@ private: void changeIO(Poco::Net::StreamSocket & socket); - bool startup(); + bool startup(Session & session); void establishSecureConnection(Int32 & payload_size, Int32 & info); @@ -64,11 +65,11 @@ private: void sendParameterStatusData(PostgreSQLProtocol::Messaging::StartupMessage & start_up_message); - void cancelRequest(); + void cancelRequest(Session & session); std::unique_ptr receiveStartupMessage(int payload_size); - void processQuery(); + void processQuery(DB::Session & session); static bool isEmptyQuery(const String & query); }; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 0339b144f09..de14f117981 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -20,9 +20,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -88,11 +90,11 @@ TCPHandler::TCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket , server(server_) , parse_proxy_protocol(parse_proxy_protocol_) , log(&Poco::Logger::get("TCPHandler")) - , connection_context(Context::createCopy(server.context())) , query_context(Context::createCopy(server.context())) , server_display_name(std::move(server_display_name_)) { } + TCPHandler::~TCPHandler() { try @@ -112,13 +114,14 @@ void TCPHandler::runImpl() setThreadName("TCPHandler"); ThreadStatus thread_status; - connection_context = Context::createCopy(server.context()); - connection_context->makeSessionContext(); + session = std::make_unique(server.context(), ClientInfo::Interface::TCP); + const auto session_context = session->sessionContext(); /// These timeouts can be changed after receiving query. + const auto & settings = session->getSettings(); - auto global_receive_timeout = connection_context->getSettingsRef().receive_timeout; - auto global_send_timeout = connection_context->getSettingsRef().send_timeout; + auto global_receive_timeout = settings.receive_timeout; + auto global_send_timeout = settings.send_timeout; socket().setReceiveTimeout(global_receive_timeout); socket().setSendTimeout(global_send_timeout); @@ -159,7 +162,7 @@ void TCPHandler::runImpl() try { /// We try to send error information to the client. - sendException(e, connection_context->getSettingsRef().calculate_text_stack_trace); + sendException(e, session->getSettings().calculate_text_stack_trace); } catch (...) {} @@ -173,20 +176,19 @@ void TCPHandler::runImpl() { Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); LOG_ERROR(log, getExceptionMessage(e, true)); - sendException(e, connection_context->getSettingsRef().calculate_text_stack_trace); + sendException(e, settings.calculate_text_stack_trace); return; } - connection_context->setCurrentDatabase(default_database); + session->setCurrentDatabase(default_database); } - Settings connection_settings = connection_context->getSettings(); - UInt64 idle_connection_timeout = connection_settings.idle_connection_timeout; - UInt64 poll_interval = connection_settings.poll_interval; + UInt64 idle_connection_timeout = settings.idle_connection_timeout; + UInt64 poll_interval = settings.poll_interval; sendHello(); - connection_context->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); + session->mutableSessionContext()->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); while (true) { @@ -209,7 +211,8 @@ void TCPHandler::runImpl() break; /// Set context of request. - query_context = Context::createCopy(connection_context); + /// TODO (nemkov): create query later in receiveQuery + query_context = session->makeQueryContext(std::string{}); // proper query_id is set later in receiveQuery Stopwatch watch; state.reset(); @@ -241,9 +244,9 @@ void TCPHandler::runImpl() * So, update some other connection settings, for flexibility. */ { - const Settings & settings = query_context->getSettingsRef(); - idle_connection_timeout = settings.idle_connection_timeout; - poll_interval = settings.poll_interval; + const Settings & query_settings = query_context->getSettingsRef(); + idle_connection_timeout = query_settings.idle_connection_timeout; + poll_interval = query_settings.poll_interval; } /** If part_uuids got received in previous packet, trying to read again. @@ -266,13 +269,13 @@ void TCPHandler::runImpl() CurrentThread::setFatalErrorCallback([this]{ sendLogs(); }); } - query_context->setExternalTablesInitializer([&connection_settings, this] (ContextPtr context) + query_context->setExternalTablesInitializer([&settings, this] (ContextPtr context) { if (context != query_context) throw Exception("Unexpected context in external tables initializer", ErrorCodes::LOGICAL_ERROR); /// Get blocks of temporary tables - readData(connection_settings); + readData(settings); /// Reset the input stream, as we received an empty block while receiving external table data. /// So, the stream has been marked as cancelled and we can't read from it anymore. @@ -303,14 +306,14 @@ void TCPHandler::runImpl() sendData(state.input_header); }); - query_context->setInputBlocksReaderCallback([&connection_settings, this] (ContextPtr context) -> Block + query_context->setInputBlocksReaderCallback([&settings, this] (ContextPtr context) -> Block { if (context != query_context) throw Exception("Unexpected context in InputBlocksReader", ErrorCodes::LOGICAL_ERROR); size_t poll_interval_ms; int receive_timeout; - std::tie(poll_interval_ms, receive_timeout) = getReadTimeouts(connection_settings); + std::tie(poll_interval_ms, receive_timeout) = getReadTimeouts(settings); if (!readDataNext(poll_interval_ms, receive_timeout)) { state.block_in.reset(); @@ -342,7 +345,7 @@ void TCPHandler::runImpl() if (state.io.out) { state.need_receive_data_for_insert = true; - processInsertQuery(connection_settings); + processInsertQuery(settings); } else if (state.need_receive_data_for_input) // It implies pipeline execution { @@ -752,12 +755,13 @@ void TCPHandler::processTablesStatusRequest() { TablesStatusRequest request; request.read(*in, client_tcp_protocol_version); + const auto session_context = session->sessionContext(); TablesStatusResponse response; for (const QualifiedTableName & table_name: request.tables) { - auto resolved_id = connection_context->tryResolveStorageID({table_name.database, table_name.table}); - StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, connection_context); + auto resolved_id = session_context->tryResolveStorageID({table_name.database, table_name.table}); + StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, session_context); if (!table) continue; @@ -924,7 +928,7 @@ bool TCPHandler::receiveProxyHeader() } LOG_TRACE(log, "Forwarded client address from PROXY header: {}", forwarded_address); - connection_context->getClientInfo().forwarded_for = forwarded_address; + session->getClientInfo().forwarded_for = forwarded_address; return true; } @@ -975,7 +979,15 @@ void TCPHandler::receiveHello() if (user != USER_INTERSERVER_MARKER) { - connection_context->setUser(user, password, socket().peerAddress()); + auto & client_info = session->getClientInfo(); + client_info.interface = ClientInfo::Interface::TCP; + client_info.client_name = client_name; + client_info.client_version_major = client_version_major; + client_info.client_version_minor = client_version_minor; + client_info.client_version_patch = client_version_patch; + client_info.client_tcp_protocol_version = client_tcp_protocol_version; + + session->setUser(user, password, socket().peerAddress()); } else { @@ -1136,7 +1148,7 @@ void TCPHandler::receiveClusterNameAndSalt() try { /// We try to send error information to the client. - sendException(e, connection_context->getSettingsRef().calculate_text_stack_trace); + sendException(e, session->getSettings().calculate_text_stack_trace); } catch (...) {} @@ -1151,6 +1163,7 @@ void TCPHandler::receiveQuery() state.is_empty = false; readStringBinary(state.query_id, *in); +// query_context = session->makeQueryContext(state.query_id); /// Client info ClientInfo & client_info = query_context->getClientInfo(); @@ -1450,7 +1463,7 @@ void TCPHandler::initBlockOutput(const Block & block) *state.maybe_compressed_out, client_tcp_protocol_version, block.cloneEmpty(), - !connection_context->getSettingsRef().low_cardinality_allow_in_native_format); + !session->getSettings().low_cardinality_allow_in_native_format); } } @@ -1463,7 +1476,7 @@ void TCPHandler::initLogsBlockOutput(const Block & block) *out, client_tcp_protocol_version, block.cloneEmpty(), - !connection_context->getSettingsRef().low_cardinality_allow_in_native_format); + !session->getSettings().low_cardinality_allow_in_native_format); } } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index 086c1f7d5e5..d8e156ee7be 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include "IServer.h" @@ -26,6 +26,7 @@ namespace Poco { class Logger; } namespace DB { +class Session; class ColumnsDescription; /// State of query processing. @@ -132,7 +133,7 @@ private: UInt64 client_version_patch = 0; UInt64 client_tcp_protocol_version = 0; - ContextMutablePtr connection_context; + std::unique_ptr session; ContextMutablePtr query_context; size_t unknown_packet_in_send_data = 0; diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index 09f9cf8b1f5..92387b13d55 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -61,8 +61,9 @@ void TableFunctionMySQL::parseArguments(const ASTPtr & ast_function, ContextPtr user_name = args[3]->as().value.safeGet(); password = args[4]->as().value.safeGet(); + const auto & settings = context->getSettingsRef(); /// Split into replicas if needed. 3306 is the default MySQL port number - size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; + const size_t max_addresses = settings.glob_expansion_max_elements; auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); pool.emplace(remote_database_name, addresses, user_name, password); From fabd7193bd687ee4b10ca826303399ff35e3d3dd Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Sun, 1 Aug 2021 17:12:34 +0300 Subject: [PATCH 182/220] Code cleanups and improvements. --- programs/local/LocalServer.cpp | 9 +- programs/server/Server.cpp | 4 +- src/Access/ContextAccess.h | 1 + src/Access/Credentials.h | 2 + src/Bridge/IBridgeHelper.cpp | 1 + src/Core/MySQL/Authentication.cpp | 16 +- src/Core/MySQL/Authentication.h | 7 +- src/Core/PostgreSQLProtocol.h | 25 +- .../ClickHouseDictionarySource.cpp | 2 +- src/IO/ReadBufferFromFileDescriptor.cpp | 1 + src/Interpreters/Context.cpp | 66 +-- src/Interpreters/Context.h | 29 +- src/Interpreters/Session.cpp | 316 ++++++------ src/Interpreters/Session.h | 89 ++-- src/Server/GRPCServer.cpp | 44 +- src/Server/HTTPHandler.cpp | 68 +-- src/Server/HTTPHandler.h | 16 +- src/Server/MySQLHandler.cpp | 52 +- src/Server/MySQLHandler.h | 2 +- src/Server/PostgreSQLHandler.cpp | 38 +- src/Server/PostgreSQLHandler.h | 7 +- src/Server/TCPHandler.cpp | 479 +++++++++--------- src/Server/TCPHandler.h | 39 +- src/TableFunctions/TableFunctionMySQL.cpp | 3 +- .../test.py | 1 + .../01455_opentelemetry_distributed.reference | 12 + .../01455_opentelemetry_distributed.sh | 30 +- 27 files changed, 677 insertions(+), 682 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 7f1bbe77d9c..44e9880fabb 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -377,11 +376,13 @@ void LocalServer::processQueries() /// we can't mutate global global_context (can lead to races, as it was already passed to some background threads) /// so we can't reuse it safely as a query context and need a copy here - Session session(global_context, ClientInfo::Interface::TCP); - session.setUser("default", "", Poco::Net::SocketAddress{}); + auto context = Context::createCopy(global_context); - auto context = session.makeQueryContext(""); + context->makeSessionContext(); + context->makeQueryContext(); + context->authenticate("default", "", Poco::Net::SocketAddress{}); + context->setCurrentQueryId(""); applyCmdSettings(context); /// Use the same query_id (and thread group) for all queries diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 98c63f9896a..c30ef52f46a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -47,13 +47,13 @@ #include #include #include -#include #include #include #include #include #include #include +#include #include #include #include @@ -1429,7 +1429,7 @@ if (ThreadFuzzer::instance().isEffective()) /// Must be done after initialization of `servers`, because async_metrics will access `servers` variable from its thread. async_metrics.start(); - Session::enableNamedSessions(); + Session::startupNamedSessions(); { String level_str = config().getString("text_log.level", ""); diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 70145b0a3ef..cde69471800 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -70,6 +70,7 @@ public: /// Returns the current user. The function can return nullptr. UserPtr getUser() const; String getUserName() const; + std::optional getUserID() const { return getParams().user_id; } /// Returns information about current and enabled roles. std::shared_ptr getRolesInfo() const; diff --git a/src/Access/Credentials.h b/src/Access/Credentials.h index 5e9fd1589e0..256ed3853ab 100644 --- a/src/Access/Credentials.h +++ b/src/Access/Credentials.h @@ -26,6 +26,8 @@ protected: String user_name; }; +/// Does not check the password/credentials and that the specified host is allowed. +/// (Used only internally in cluster, if the secret matches) class AlwaysAllowCredentials : public Credentials { diff --git a/src/Bridge/IBridgeHelper.cpp b/src/Bridge/IBridgeHelper.cpp index 5c884a2ca3d..984641be3d2 100644 --- a/src/Bridge/IBridgeHelper.cpp +++ b/src/Bridge/IBridgeHelper.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace fs = std::filesystem; diff --git a/src/Core/MySQL/Authentication.cpp b/src/Core/MySQL/Authentication.cpp index bc34b5637d6..aeb9a411082 100644 --- a/src/Core/MySQL/Authentication.cpp +++ b/src/Core/MySQL/Authentication.cpp @@ -2,8 +2,6 @@ #include #include #include -#include -#include #include #include @@ -74,7 +72,7 @@ Native41::Native41(const String & password, const String & auth_plugin_data) } void Native41::authenticate( - const String & user_name, std::optional auth_response, Session & session, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool, const Poco::Net::SocketAddress & address) { if (!auth_response) @@ -87,7 +85,7 @@ void Native41::authenticate( if (auth_response->empty()) { - session.setUser(user_name, "", address); + session.authenticate(user_name, "", address); return; } @@ -97,9 +95,7 @@ void Native41::authenticate( + " bytes, received: " + std::to_string(auth_response->size()) + " bytes.", ErrorCodes::UNKNOWN_EXCEPTION); - const auto user_authentication = session.getUserAuthentication(user_name); - - Poco::SHA1Engine::Digest double_sha1_value = user_authentication.getPasswordDoubleSHA1(); + Poco::SHA1Engine::Digest double_sha1_value = session.getPasswordDoubleSHA1(user_name); assert(double_sha1_value.size() == Poco::SHA1Engine::DIGEST_SIZE); Poco::SHA1Engine engine; @@ -112,7 +108,7 @@ void Native41::authenticate( { password_sha1[i] = digest[i] ^ static_cast((*auth_response)[i]); } - session.setUser(user_name, password_sha1, address); + session.authenticate(user_name, password_sha1, address); } #if USE_SSL @@ -137,7 +133,7 @@ Sha256Password::Sha256Password(RSA & public_key_, RSA & private_key_, Poco::Logg } void Sha256Password::authenticate( - const String & user_name, std::optional auth_response, Session & session, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) { if (!auth_response) @@ -232,7 +228,7 @@ void Sha256Password::authenticate( password.pop_back(); } - session.setUser(user_name, password, address); + session.authenticate(user_name, password, address); } #endif diff --git a/src/Core/MySQL/Authentication.h b/src/Core/MySQL/Authentication.h index 0dde8d10c0e..a60e769434e 100644 --- a/src/Core/MySQL/Authentication.h +++ b/src/Core/MySQL/Authentication.h @@ -15,6 +15,7 @@ namespace DB { +class Session; namespace MySQLProtocol { @@ -32,7 +33,7 @@ public: virtual String getAuthPluginData() = 0; virtual void authenticate( - const String & user_name, std::optional auth_response, Session & session, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) = 0; }; @@ -49,7 +50,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, Session & session, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool /* is_secure_connection */, const Poco::Net::SocketAddress & address) override; private: @@ -69,7 +70,7 @@ public: String getAuthPluginData() override { return scramble; } void authenticate( - const String & user_name, std::optional auth_response, Session & session, + const String & user_name, Session & session, std::optional auth_response, std::shared_ptr packet_endpoint, bool is_secure_connection, const Poco::Net::SocketAddress & address) override; private: diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 19bcc727105..aef0ed6ab25 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -1,14 +1,11 @@ #pragma once -#include -#include #include -#include -#include #include #include #include #include +#include #include #include #include @@ -808,8 +805,9 @@ protected: Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { - try { - session.setUser(user_name, password, address); + try + { + session.authenticate(user_name, password, address); } catch (const Exception &) { @@ -841,7 +839,7 @@ public: Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) override { - setPassword(user_name, "", session, mt, address); + return setPassword(user_name, "", session, mt, address); } Authentication::Type getType() const override @@ -865,7 +863,7 @@ public: if (type == Messaging::FrontMessageType::PASSWORD_MESSAGE) { std::unique_ptr password = mt.receive(); - setPassword(user_name, password->password, session, mt, address); + return setPassword(user_name, password->password, session, mt, address); } else throw Exception( @@ -902,16 +900,7 @@ public: Messaging::MessageTransport & mt, const Poco::Net::SocketAddress & address) { - Authentication::Type user_auth_type; - try - { - user_auth_type = session.getUserAuthentication(user_name).getType(); - } - catch (const std::exception & e) - { - session.onLogInFailure(user_name, e); - throw; - } + Authentication::Type user_auth_type = session.getAuthenticationType(user_name); if (type_to_method.find(user_auth_type) != type_to_method.end()) { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 0f085a7c1a2..d4f01dee8b2 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -255,7 +255,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). if (configuration.is_local) { - context_copy->setUser(configuration.user, configuration.password, Poco::Net::SocketAddress("127.0.0.1", 0)); + context_copy->authenticate(configuration.user, configuration.password, Poco::Net::SocketAddress("127.0.0.1", 0)); context_copy = copyContextAndApplySettings(config_prefix, context_copy, config); } diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index fdb538d4a49..e60ec335ca1 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -12,6 +12,7 @@ #include #include #include +#include namespace ProfileEvents diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 601127c99b5..4d918d0fbb6 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -588,48 +588,45 @@ ConfigurationPtr Context::getUsersConfig() } -void Context::setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address) +void Context::authenticate(const String & name, const String & password, const Poco::Net::SocketAddress & address) { - auto lock = getLock(); + authenticate(BasicCredentials(name, password), address); +} + +void Context::authenticate(const Credentials & credentials, const Poco::Net::SocketAddress & address) +{ + auto authenticated_user_id = getAccessControlManager().login(credentials, address.host()); client_info.current_user = credentials.getUserName(); client_info.current_address = address; #if defined(ARCADIA_BUILD) /// This is harmful field that is used only in foreign "Arcadia" build. - client_info.current_password.clear(); if (const auto * basic_credentials = dynamic_cast(&credentials)) client_info.current_password = basic_credentials->getPassword(); #endif - /// Find a user with such name and check the credentials. - auto new_user_id = getAccessControlManager().login(credentials, address.host()); - auto new_access = getAccessControlManager().getContextAccess( - new_user_id, /* current_roles = */ {}, /* use_default_roles = */ true, - settings, current_database, client_info); + setUser(authenticated_user_id); +} - user_id = new_user_id; - access = std::move(new_access); +void Context::setUser(const UUID & user_id_) +{ + auto lock = getLock(); + + user_id = user_id_; + + access = getAccessControlManager().getContextAccess( + user_id_, /* current_roles = */ {}, /* use_default_roles = */ true, settings, current_database, client_info); auto user = access->getUser(); current_roles = std::make_shared>(user->granted_roles.findGranted(user->default_roles)); - if (!user->default_database.empty()) - setCurrentDatabase(user->default_database); - auto default_profile_info = access->getDefaultProfileInfo(); settings_constraints_and_current_profiles = default_profile_info->getConstraintsAndProfileIDs(); applySettingsChanges(default_profile_info->settings); -} -void Context::setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address) -{ - setUser(BasicCredentials(name, password), address); -} - -void Context::setUserWithoutCheckingPassword(const String & name, const Poco::Net::SocketAddress & address) -{ - setUser(AlwaysAllowCredentials(name), address); + if (!user->default_database.empty()) + setCurrentDatabase(user->default_database); } std::shared_ptr Context::getUser() const @@ -637,12 +634,6 @@ std::shared_ptr Context::getUser() const return getAccess()->getUser(); } -void Context::setQuotaKey(String quota_key_) -{ - auto lock = getLock(); - client_info.quota_key = std::move(quota_key_); -} - String Context::getUserName() const { return getAccess()->getUserName(); @@ -655,6 +646,13 @@ std::optional Context::getUserID() const } +void Context::setQuotaKey(String quota_key_) +{ + auto lock = getLock(); + client_info.quota_key = std::move(quota_key_); +} + + void Context::setCurrentRoles(const std::vector & current_roles_) { auto lock = getLock(); @@ -736,10 +734,13 @@ ASTPtr Context::getRowPolicyCondition(const String & database, const String & ta void Context::setInitialRowPolicy() { auto lock = getLock(); - auto initial_user_id = getAccessControlManager().find(client_info.initial_user); initial_row_policy = nullptr; - if (initial_user_id) - initial_row_policy = getAccessControlManager().getEnabledRowPolicies(*initial_user_id, {}); + if (client_info.initial_user == client_info.current_user) + return; + auto initial_user_id = getAccessControlManager().find(client_info.initial_user); + if (!initial_user_id) + return; + initial_row_policy = getAccessControlManager().getEnabledRowPolicies(*initial_user_id, {}); } @@ -1180,6 +1181,9 @@ void Context::setCurrentQueryId(const String & query_id) } client_info.current_query_id = query_id_to_set; + + if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + client_info.initial_query_id = client_info.current_query_id; } void Context::killCurrentQuery() diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 0bb32bb7b43..4e378dacf01 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -14,21 +14,16 @@ #include #include #include -#include #include #if !defined(ARCADIA_BUILD) # include "config_core.h" #endif -#include -#include -#include #include #include #include #include -#include namespace Poco::Net { class IPAddress; } @@ -67,6 +62,7 @@ class ProcessList; class QueryStatus; class Macros; struct Progress; +struct FileProgress; class Clusters; class QueryLog; class QueryThreadLog; @@ -366,23 +362,21 @@ public: void setUsersConfig(const ConfigurationPtr & config); ConfigurationPtr getUsersConfig(); - /// Sets the current user, checks the credentials and that the specified host is allowed. - /// Must be called before getClientInfo() can be called. - void setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address); - void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address); + /// Sets the current user, checks the credentials and that the specified address is allowed to connect from. + /// The function throws an exception if there is no such user or password is wrong. + void authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address); + void authenticate(const Credentials & credentials, const Poco::Net::SocketAddress & address); - /// Sets the current user, *does not check the password/credentials and that the specified host is allowed*. - /// Must be called before getClientInfo. - /// - /// (Used only internally in cluster, if the secret matches) - void setUserWithoutCheckingPassword(const String & name, const Poco::Net::SocketAddress & address); - - void setQuotaKey(String quota_key_); + /// Sets the current user assuming that he/she is already authenticated. + /// WARNING: This function doesn't check password! Don't use until it's necessary! + void setUser(const UUID & user_id_); UserPtr getUser() const; String getUserName() const; std::optional getUserID() const; + void setQuotaKey(String quota_key_); + void setCurrentRoles(const std::vector & current_roles_); void setCurrentRolesDefault(); boost::container::flat_set getCurrentRoles() const; @@ -590,8 +584,6 @@ public: std::optional getTCPPortSecure() const; - std::shared_ptr acquireNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); - /// For methods below you may need to acquire the context lock by yourself. ContextMutablePtr getQueryContext() const; @@ -602,7 +594,6 @@ public: bool hasSessionContext() const { return !session_context.expired(); } ContextMutablePtr getGlobalContext() const; - bool hasGlobalContext() const { return !global_context.expired(); } bool isGlobalContext() const { diff --git a/src/Interpreters/Session.cpp b/src/Interpreters/Session.cpp index acebc182a64..7334f2e7640 100644 --- a/src/Interpreters/Session.cpp +++ b/src/Interpreters/Session.cpp @@ -1,24 +1,22 @@ #include #include -#include #include +#include #include #include #include #include #include -#include - -#include -#include -#include #include -#include +#include #include +#include +#include #include + namespace DB { @@ -27,13 +25,13 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int SESSION_NOT_FOUND; extern const int SESSION_IS_LOCKED; - extern const int NOT_IMPLEMENTED; } + class NamedSessionsStorage; -/// User name and session identifier. Named sessions are local to users. -using NamedSessionKey = std::pair; +/// User ID and session identifier. Named sessions are local to users. +using NamedSessionKey = std::pair; /// Named sessions. The user could specify session identifier to reuse settings and temporary tables in subsequent requests. struct NamedSessionData @@ -75,21 +73,16 @@ public: } /// Find existing session or create a new. - std::shared_ptr acquireSession( + std::pair, bool> acquireSession( + const ContextPtr & global_context, + const UUID & user_id, const String & session_id, - ContextMutablePtr context, std::chrono::steady_clock::duration timeout, bool throw_if_not_found) { std::unique_lock lock(mutex); - const auto & client_info = context->getClientInfo(); - const auto & user_name = client_info.current_user; - - if (user_name.empty()) - throw Exception("Empty user name.", ErrorCodes::LOGICAL_ERROR); - - Key key(user_name, session_id); + Key key{user_id, session_id}; auto it = sessions.find(key); if (it == sessions.end()) @@ -98,22 +91,20 @@ public: throw Exception("Session not found.", ErrorCodes::SESSION_NOT_FOUND); /// Create a new session from current context. + auto context = Context::createCopy(global_context); it = sessions.insert(std::make_pair(key, std::make_shared(key, context, timeout, *this))).first; + const auto & session = it->second; + return {session, true}; } - else if (it->second->key.first != client_info.current_user) + else { - throw Exception("Session belongs to a different user", ErrorCodes::SESSION_IS_LOCKED); + /// Use existing session. + const auto & session = it->second; + + if (!session.unique()) + throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); + return {session, false}; } - - /// Use existing session. - const auto & session = it->second; - - if (!session.unique()) - throw Exception("Session is locked by a concurrent client.", ErrorCodes::SESSION_IS_LOCKED); - - session->context->getClientInfo() = client_info; - - return session; } void releaseSession(NamedSessionData & session) @@ -229,164 +220,195 @@ void NamedSessionData::release() std::optional Session::named_sessions = std::nullopt; -void Session::enableNamedSessions() +void Session::startupNamedSessions() { named_sessions.emplace(); } -Session::Session(const ContextPtr & context_to_copy, ClientInfo::Interface interface, std::optional default_format) - : session_context(Context::createCopy(context_to_copy)), - initial_session_context(session_context) +Session::Session(const ContextPtr & global_context_, ClientInfo::Interface interface_) + : global_context(global_context_) { - session_context->makeSessionContext(); - session_context->getClientInfo().interface = interface; - - if (default_format) - session_context->setDefaultFormat(*default_format); + prepared_client_info.emplace(); + prepared_client_info->interface = interface_; } Session::Session(Session &&) = default; Session::~Session() { - releaseNamedSession(); - - if (access) - { - auto user = access->getUser(); - if (user) - onLogOut(); - } -} - -Authentication Session::getUserAuthentication(const String & user_name) const -{ - return session_context->getAccessControlManager().read(user_name)->authentication; -} - -void Session::setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address) -{ - try - { - session_context->setUser(credentials, address); - - // Caching access just in case if context is going to be replaced later (e.g. with context of NamedSessionData) - access = session_context->getAccess(); - - // Check if this is a not an intercluster session, but the real one. - if (access && access->getUser() && dynamic_cast(&credentials)) - { - onLogInSuccess(); - } - } - catch (const std::exception & e) - { - onLogInFailure(credentials.getUserName(), e); - throw; - } -} - -void Session::setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address) -{ - setUser(BasicCredentials(name, password), address); -} - -void Session::onLogInSuccess() -{ -} - -void Session::onLogInFailure(const String & /* user_name */, const std::exception & /* failure_reason */) -{ -} - -void Session::onLogOut() -{ -} - -void Session::promoteToNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check) -{ - if (!named_sessions) - throw Exception("Support for named sessions is not enabled", ErrorCodes::NOT_IMPLEMENTED); - - auto new_named_session = named_sessions->acquireSession(session_id, session_context, timeout, session_check); - - // Must retain previous client info cause otherwise source client address and port, - // and other stuff are reused from previous user of the said session. - const ClientInfo prev_client_info = session_context->getClientInfo(); - - session_context = new_named_session->context; - session_context->getClientInfo() = prev_client_info; - session_context->makeSessionContext(); - - named_session.swap(new_named_session); -} - -/// Early release a NamedSessionData. -void Session::releaseNamedSession() -{ + /// Early release a NamedSessionData. if (named_session) - { named_session->release(); - named_session.reset(); - } - - session_context = initial_session_context; } -ContextMutablePtr Session::makeQueryContext(const String & query_id) const +Authentication::Type Session::getAuthenticationType(const String & user_name) const { - ContextMutablePtr new_query_context = Context::createCopy(session_context); - - new_query_context->setCurrentQueryId(query_id); - new_query_context->setSessionContext(session_context); - new_query_context->makeQueryContext(); - - ClientInfo & client_info = new_query_context->getClientInfo(); - client_info.initial_user = client_info.current_user; - client_info.initial_query_id = client_info.current_query_id; - client_info.initial_address = client_info.current_address; - - return new_query_context; + return global_context->getAccessControlManager().read(user_name)->authentication.getType(); } -ContextPtr Session::sessionContext() const +Authentication::Digest Session::getPasswordDoubleSHA1(const String & user_name) const { - return session_context; + return global_context->getAccessControlManager().read(user_name)->authentication.getPasswordDoubleSHA1(); } -ContextMutablePtr Session::mutableSessionContext() +void Session::authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address) { - return session_context; + authenticate(BasicCredentials{user_name, password}, address); +} + +void Session::authenticate(const Credentials & credentials_, const Poco::Net::SocketAddress & address_) +{ + if (session_context) + throw Exception("If there is a session context it must be created after authentication", ErrorCodes::LOGICAL_ERROR); + + user_id = global_context->getAccessControlManager().login(credentials_, address_.host()); + + prepared_client_info->current_user = credentials_.getUserName(); + prepared_client_info->current_address = address_; + +#if defined(ARCADIA_BUILD) + /// This is harmful field that is used only in foreign "Arcadia" build. + if (const auto * basic_credentials = dynamic_cast(&credentials_)) + session_client_info->current_password = basic_credentials->getPassword(); +#endif } ClientInfo & Session::getClientInfo() { - return session_context->getClientInfo(); + return session_context ? session_context->getClientInfo() : *prepared_client_info; } const ClientInfo & Session::getClientInfo() const { - return session_context->getClientInfo(); + return session_context ? session_context->getClientInfo() : *prepared_client_info; } -const Settings & Session::getSettings() const +ContextMutablePtr Session::makeSessionContext() { - return session_context->getSettingsRef(); + if (session_context) + throw Exception("Session context already exists", ErrorCodes::LOGICAL_ERROR); + if (query_context_created) + throw Exception("Session context must be created before any query context", ErrorCodes::LOGICAL_ERROR); + + /// Make a new session context. + ContextMutablePtr new_session_context; + new_session_context = Context::createCopy(global_context); + new_session_context->makeSessionContext(); + + /// Copy prepared client info to the new session context. + auto & res_client_info = new_session_context->getClientInfo(); + res_client_info = std::move(prepared_client_info).value(); + prepared_client_info.reset(); + + /// Set user information for the new context: current profiles, roles, access rights. + if (user_id) + new_session_context->setUser(*user_id); + + /// Session context is ready. + session_context = new_session_context; + user = session_context->getUser(); + + return session_context; } -void Session::setQuotaKey(const String & quota_key) +ContextMutablePtr Session::makeSessionContext(const String & session_id_, std::chrono::steady_clock::duration timeout_, bool session_check_) { - session_context->setQuotaKey(quota_key); + if (session_context) + throw Exception("Session context already exists", ErrorCodes::LOGICAL_ERROR); + if (query_context_created) + throw Exception("Session context must be created before any query context", ErrorCodes::LOGICAL_ERROR); + if (!named_sessions) + throw Exception("Support for named sessions is not enabled", ErrorCodes::LOGICAL_ERROR); + + /// Make a new session context OR + /// if the `session_id` and `user_id` were used before then just get a previously created session context. + std::shared_ptr new_named_session; + bool new_named_session_created = false; + std::tie(new_named_session, new_named_session_created) + = named_sessions->acquireSession(global_context, user_id.value_or(UUID{}), session_id_, timeout_, session_check_); + + auto new_session_context = new_named_session->context; + new_session_context->makeSessionContext(); + + /// Copy prepared client info to the session context, no matter it's been just created or not. + /// If we continue using a previously created session context found by session ID + /// it's necessary to replace the client info in it anyway, because it contains actual connection information (client address, etc.) + auto & res_client_info = new_session_context->getClientInfo(); + res_client_info = std::move(prepared_client_info).value(); + prepared_client_info.reset(); + + /// Set user information for the new context: current profiles, roles, access rights. + if (user_id && !new_session_context->getUser()) + new_session_context->setUser(*user_id); + + /// Session context is ready. + session_context = new_session_context; + session_id = session_id_; + named_session = new_named_session; + named_session_created = new_named_session_created; + user = session_context->getUser(); + + return session_context; } -String Session::getCurrentDatabase() const +ContextMutablePtr Session::makeQueryContext(const ClientInfo & query_client_info) const { - return session_context->getCurrentDatabase(); + return makeQueryContextImpl(&query_client_info, nullptr); } -void Session::setCurrentDatabase(const String & name) +ContextMutablePtr Session::makeQueryContext(ClientInfo && query_client_info) const { - session_context->setCurrentDatabase(name); + return makeQueryContextImpl(nullptr, &query_client_info); +} + +ContextMutablePtr Session::makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const +{ + /// We can create a query context either from a session context or from a global context. + bool from_session_context = static_cast(session_context); + + /// Create a new query context. + ContextMutablePtr query_context = Context::createCopy(from_session_context ? session_context : global_context); + query_context->makeQueryContext(); + + /// Copy the specified client info to the new query context. + auto & res_client_info = query_context->getClientInfo(); + if (client_info_to_move) + res_client_info = std::move(*client_info_to_move); + else if (client_info_to_copy && (client_info_to_copy != &getClientInfo())) + res_client_info = *client_info_to_copy; + + /// Copy current user's name and address if it was authenticated after query_client_info was initialized. + if (prepared_client_info && !prepared_client_info->current_user.empty()) + { + res_client_info.current_user = prepared_client_info->current_user; + res_client_info.current_address = prepared_client_info->current_address; +#if defined(ARCADIA_BUILD) + res_client_info.current_password = prepared_client_info->current_password; +#endif + } + + /// Set parameters of initial query. + if (res_client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) + res_client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + + if (res_client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + { + res_client_info.initial_user = res_client_info.current_user; + res_client_info.initial_address = res_client_info.current_address; + } + + /// Sets that row policies from the initial user should be used too. + query_context->setInitialRowPolicy(); + + /// Set user information for the new context: current profiles, roles, access rights. + if (user_id && !query_context->getUser()) + query_context->setUser(*user_id); + + /// Query context is ready. + query_context_created = true; + user = query_context->getUser(); + + return query_context; } } diff --git a/src/Interpreters/Session.h b/src/Interpreters/Session.h index 300ed779c49..58370aad2d0 100644 --- a/src/Interpreters/Session.h +++ b/src/Interpreters/Session.h @@ -1,8 +1,9 @@ #pragma once -#include -#include +#include +#include #include +#include #include #include @@ -13,77 +14,77 @@ namespace Poco::Net { class SocketAddress; } namespace DB { class Credentials; -class ContextAccess; -struct Settings; class Authentication; struct NamedSessionData; class NamedSessionsStorage; +struct User; +using UserPtr = std::shared_ptr; /** Represents user-session from the server perspective, * basically it is just a smaller subset of Context API, simplifies Context management. * * Holds session context, facilitates acquisition of NamedSession and proper creation of query contexts. - * Adds log in, log out and login failure events to the SessionLog. */ class Session { - static std::optional named_sessions; - public: /// Allow to use named sessions. The thread will be run to cleanup sessions after timeout has expired. /// The method must be called at the server startup. - static void enableNamedSessions(); + static void startupNamedSessions(); -// static Session makeSessionFromCopyOfContext(const ContextPtr & _context_to_copy); - Session(const ContextPtr & context_to_copy, ClientInfo::Interface interface, std::optional default_format = std::nullopt); - virtual ~Session(); + Session(const ContextPtr & global_context_, ClientInfo::Interface interface_); + Session(Session &&); + ~Session(); Session(const Session &) = delete; Session& operator=(const Session &) = delete; - Session(Session &&); -// Session& operator=(Session &&); + /// Provides information about the authentication type of a specified user. + Authentication::Type getAuthenticationType(const String & user_name) const; + Authentication::Digest getPasswordDoubleSHA1(const String & user_name) const; - Authentication getUserAuthentication(const String & user_name) const; - void setUser(const Credentials & credentials, const Poco::Net::SocketAddress & address); - void setUser(const String & name, const String & password, const Poco::Net::SocketAddress & address); - - /// Handle login and logout events. - void onLogInSuccess(); - void onLogInFailure(const String & user_name, const std::exception & /* failure_reason */); - void onLogOut(); - - /** Propmotes current session to a named session. - * - * that is: re-uses or creates NamedSession and then piggybacks on it's context, - * retaining ClientInfo of current session_context. - * Acquired named_session is then released in the destructor. - */ - void promoteToNamedSession(const String & session_id, std::chrono::steady_clock::duration timeout, bool session_check); - /// Early release a NamedSession. - void releaseNamedSession(); - - ContextMutablePtr makeQueryContext(const String & query_id) const; - - ContextPtr sessionContext() const; - ContextMutablePtr mutableSessionContext(); + /// Sets the current user, checks the credentials and that the specified address is allowed to connect from. + /// The function throws an exception if there is no such user or password is wrong. + void authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address); + void authenticate(const Credentials & credentials_, const Poco::Net::SocketAddress & address_); + /// Returns a reference to session ClientInfo. ClientInfo & getClientInfo(); const ClientInfo & getClientInfo() const; - const Settings & getSettings() const; + /// Makes a session context, can be used one or zero times. + /// The function also assigns an user to this context. + ContextMutablePtr makeSessionContext(); + ContextMutablePtr makeSessionContext(const String & session_id_, std::chrono::steady_clock::duration timeout_, bool session_check_); + ContextMutablePtr sessionContext() { return session_context; } + ContextPtr sessionContext() const { return session_context; } - void setQuotaKey(const String & quota_key); - - String getCurrentDatabase() const; - void setCurrentDatabase(const String & name); + /// Makes a query context, can be used multiple times, with or without makeSession() called earlier. + /// The query context will be created from a copy of a session context if it exists, or from a copy of + /// a global context otherwise. In the latter case the function also assigns an user to this context. + ContextMutablePtr makeQueryContext() const { return makeQueryContext(getClientInfo()); } + ContextMutablePtr makeQueryContext(const ClientInfo & query_client_info) const; + ContextMutablePtr makeQueryContext(ClientInfo && query_client_info) const; private: + ContextMutablePtr makeQueryContextImpl(const ClientInfo * client_info_to_copy, ClientInfo * client_info_to_move) const; + + const ContextPtr global_context; + + /// ClientInfo that will be copied to a session context when it's created. + std::optional prepared_client_info; + + mutable UserPtr user; + std::optional user_id; + ContextMutablePtr session_context; - // So that Session can be used after forced release of named_session. - const ContextMutablePtr initial_session_context; - std::shared_ptr access; + mutable bool query_context_created = false; + + String session_id; std::shared_ptr named_session; + bool named_session_created = false; + + static std::optional named_sessions; }; } diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index f03d0ae5f9f..f0c6e208323 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -11,9 +11,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -55,7 +55,6 @@ namespace ErrorCodes extern const int NETWORK_ERROR; extern const int NO_DATA_TO_INSERT; extern const int SUPPORT_IS_DISABLED; - extern const int UNKNOWN_DATABASE; } namespace @@ -561,7 +560,7 @@ namespace IServer & iserver; Poco::Logger * log = nullptr; - std::shared_ptr session; + std::optional session; ContextMutablePtr query_context; std::optional query_scope; String query_text; @@ -690,32 +689,20 @@ namespace password = ""; } - /// Create context. - session = std::make_shared(iserver.context(), ClientInfo::Interface::GRPC); /// Authentication. - session->setUser(user, password, user_address); - if (!quota_key.empty()) - session->setQuotaKey(quota_key); + session.emplace(iserver.context(), ClientInfo::Interface::GRPC); + session->authenticate(user, password, user_address); + session->getClientInfo().quota_key = quota_key; /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. if (!query_info.session_id().empty()) { - session->promoteToNamedSession( - query_info.session_id(), - getSessionTimeout(query_info, iserver.config()), - query_info.session_check()); + session->makeSessionContext( + query_info.session_id(), getSessionTimeout(query_info, iserver.config()), query_info.session_check()); } - query_context = session->makeQueryContext(query_info.query_id()); - query_scope.emplace(query_context); - - /// Set client info. - ClientInfo & client_info = query_context->getClientInfo(); - client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.initial_user = client_info.current_user; - client_info.initial_query_id = client_info.current_query_id; - client_info.initial_address = client_info.current_address; + query_context = session->makeQueryContext(); /// Prepare settings. SettingsChanges settings_changes; @@ -725,11 +712,14 @@ namespace } query_context->checkSettingsConstraints(settings_changes); query_context->applySettingsChanges(settings_changes); - const Settings & settings = query_context->getSettingsRef(); + + query_context->setCurrentQueryId(query_info.query_id()); + query_scope.emplace(query_context); /// Prepare for sending exceptions and logs. - send_exception_with_stacktrace = query_context->getSettingsRef().calculate_text_stack_trace; - const auto client_logs_level = query_context->getSettingsRef().send_logs_level; + const Settings & settings = query_context->getSettingsRef(); + send_exception_with_stacktrace = settings.calculate_text_stack_trace; + const auto client_logs_level = settings.send_logs_level; if (client_logs_level != LogsLevel::none) { logs_queue = std::make_shared(); @@ -740,14 +730,10 @@ namespace /// Set the current database if specified. if (!query_info.database().empty()) - { - if (!DatabaseCatalog::instance().isDatabaseExist(query_info.database())) - throw Exception("Database " + query_info.database() + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); query_context->setCurrentDatabase(query_info.database()); - } /// The interactive delay will be used to show progress. - interactive_delay = query_context->getSettingsRef().interactive_delay; + interactive_delay = settings.interactive_delay; query_context->setProgressCallback([this](const Progress & value) { return progress.incrementPiecewiseAtomically(value); }); /// Parse the query. diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 0e6b7d57b7c..0492b58dc88 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -19,9 +19,9 @@ #include #include #include -#include #include #include +#include #include #include #include @@ -262,6 +262,7 @@ void HTTPHandler::pushDelayedResults(Output & used_output) HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) : server(server_) , log(&Poco::Logger::get(name)) + , default_settings(server.context()->getSettingsRef()) { server_display_name = server.config().getString("display_name", getFQDNOrHostName()); } @@ -269,10 +270,7 @@ HTTPHandler::HTTPHandler(IServer & server_, const std::string & name) /// We need d-tor to be present in this translation unit to make it play well with some /// forward decls in the header. Other than that, the default d-tor would be OK. -HTTPHandler::~HTTPHandler() -{ - (void)this; -} +HTTPHandler::~HTTPHandler() = default; bool HTTPHandler::authenticateUser( @@ -352,7 +350,7 @@ bool HTTPHandler::authenticateUser( else { if (!request_credentials) - request_credentials = request_session->sessionContext()->makeGSSAcceptorContext(); + request_credentials = server.context()->makeGSSAcceptorContext(); auto * gss_acceptor_context = dynamic_cast(request_credentials.get()); if (!gss_acceptor_context) @@ -378,9 +376,7 @@ bool HTTPHandler::authenticateUser( } /// Set client info. It will be used for quota accounting parameters in 'setUser' method. - - ClientInfo & client_info = request_session->getClientInfo(); - client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + ClientInfo & client_info = session->getClientInfo(); ClientInfo::HTTPMethod http_method = ClientInfo::HTTPMethod::UNKNOWN; if (request.getMethod() == HTTPServerRequest::HTTP_GET) @@ -392,10 +388,11 @@ bool HTTPHandler::authenticateUser( client_info.http_user_agent = request.get("User-Agent", ""); client_info.http_referer = request.get("Referer", ""); client_info.forwarded_for = request.get("X-Forwarded-For", ""); + client_info.quota_key = quota_key; try { - request_session->setUser(*request_credentials, request.clientAddress()); + session->authenticate(*request_credentials, request.clientAddress()); } catch (const Authentication::Require & required_credentials) { @@ -412,7 +409,7 @@ bool HTTPHandler::authenticateUser( } catch (const Authentication::Require & required_credentials) { - request_credentials = request_session->sessionContext()->makeGSSAcceptorContext(); + request_credentials = server.context()->makeGSSAcceptorContext(); if (required_credentials.getRealm().empty()) response.set("WWW-Authenticate", "Negotiate"); @@ -425,14 +422,6 @@ bool HTTPHandler::authenticateUser( } request_credentials.reset(); - - if (!quota_key.empty()) - request_session->setQuotaKey(quota_key); - - /// Query sent through HTTP interface is initial. - client_info.initial_user = client_info.current_user; - client_info.initial_address = client_info.current_address; - return true; } @@ -463,20 +452,16 @@ void HTTPHandler::processQuery( session_id = params.get("session_id"); session_timeout = parseSessionTimeout(config, params); std::string session_check = params.get("session_check", ""); - request_session->promoteToNamedSession(session_id, session_timeout, session_check == "1"); + session->makeSessionContext(session_id, session_timeout, session_check == "1"); } - SCOPE_EXIT({ - request_session->releaseNamedSession(); - }); - // Parse the OpenTelemetry traceparent header. // Disable in Arcadia -- it interferes with the // test_clickhouse.TestTracing.test_tracing_via_http_proxy[traceparent] test. + ClientInfo client_info = session->getClientInfo(); #if !defined(ARCADIA_BUILD) if (request.has("traceparent")) { - ClientInfo & client_info = request_session->getClientInfo(); std::string opentelemetry_traceparent = request.get("traceparent"); std::string error; if (!client_info.client_trace_context.parseTraceparentHeader( @@ -486,16 +471,11 @@ void HTTPHandler::processQuery( "Failed to parse OpenTelemetry traceparent header '{}': {}", opentelemetry_traceparent, error); } - client_info.client_trace_context.tracestate = request.get("tracestate", ""); } #endif - // Set the query id supplied by the user, if any, and also update the OpenTelemetry fields. - auto context = request_session->makeQueryContext(params.get("query_id", request.get("X-ClickHouse-Query-Id", ""))); - - ClientInfo & client_info = context->getClientInfo(); - client_info.initial_query_id = client_info.current_query_id; + auto context = session->makeQueryContext(std::move(client_info)); /// The client can pass a HTTP header indicating supported compression method (gzip or deflate). String http_response_compression_methods = request.get("Accept-Encoding", ""); @@ -560,7 +540,7 @@ void HTTPHandler::processQuery( if (buffer_until_eof) { - const std::string tmp_path(context->getTemporaryVolume()->getDisk()->getPath()); + const std::string tmp_path(server.context()->getTemporaryVolume()->getDisk()->getPath()); const std::string tmp_path_template(tmp_path + "http_buffers/"); auto create_tmp_disk_buffer = [tmp_path_template] (const WriteBufferPtr &) @@ -706,6 +686,9 @@ void HTTPHandler::processQuery( context->checkSettingsConstraints(settings_changes); context->applySettingsChanges(settings_changes); + // Set the query id supplied by the user, if any, and also update the OpenTelemetry fields. + context->setCurrentQueryId(params.get("query_id", request.get("X-ClickHouse-Query-Id", ""))); + const auto & query = getQuery(request, params, context); std::unique_ptr in_param = std::make_unique(query); in = has_external_data ? std::move(in_param) : std::make_unique(*in_param, *in_post_maybe_compressed); @@ -856,23 +839,10 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse setThreadName("HTTPHandler"); ThreadStatus thread_status; - SCOPE_EXIT({ - // If there is no request_credentials instance waiting for the next round, then the request is processed, - // so no need to preserve request_session either. - // Needs to be performed with respect to the other destructors in the scope though. - if (!request_credentials) - request_session.reset(); - }); - - if (!request_session) - { - // Context should be initialized before anything, for correct memory accounting. - request_session = std::make_shared(server.context(), ClientInfo::Interface::HTTP); - request_credentials.reset(); - } - - /// Cannot be set here, since query_id is unknown. + session = std::make_unique(server.context(), ClientInfo::Interface::HTTP); + SCOPE_EXIT({ session.reset(); }); std::optional query_scope; + Output used_output; /// In case of exception, send stack trace to client. @@ -886,7 +856,7 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse if (request.getVersion() == HTTPServerRequest::HTTP_1_1) response.setChunkedTransferEncoding(true); - HTMLForm params(request_session->getSettings(), request); + HTMLForm params(default_settings, request); with_stacktrace = params.getParsed("stacktrace", false); /// FIXME: maybe this check is already unnecessary. diff --git a/src/Server/HTTPHandler.h b/src/Server/HTTPHandler.h index bca73ca7cb8..98f573f8cef 100644 --- a/src/Server/HTTPHandler.h +++ b/src/Server/HTTPHandler.h @@ -21,6 +21,7 @@ namespace DB class Session; class Credentials; class IServer; +struct Settings; class WriteBufferFromHTTPServerResponse; using CompiledRegexPtr = std::shared_ptr; @@ -72,15 +73,22 @@ private: CurrentMetrics::Increment metric_increment{CurrentMetrics::HTTPConnection}; - // The request_session and the request_credentials instances may outlive a single request/response loop. + /// Reference to the immutable settings in the global context. + /// Those settings are used only to extract a http request's parameters. + /// See settings http_max_fields, http_max_field_name_size, http_max_field_value_size in HTMLForm. + const Settings & default_settings; + + // session is reset at the end of each request/response. + std::unique_ptr session; + + // The request_credential instance may outlive a single request/response loop. // This happens only when the authentication mechanism requires more than a single request/response exchange (e.g., SPNEGO). - std::shared_ptr request_session; std::unique_ptr request_credentials; // Returns true when the user successfully authenticated, - // the request_session instance will be configured accordingly, and the request_credentials instance will be dropped. + // the session instance will be configured accordingly, and the request_credentials instance will be dropped. // Returns false when the user is not authenticated yet, and the 'Negotiate' response is sent, - // the request_session and request_credentials instances are preserved. + // the session and request_credentials instances are preserved. // Throws an exception if authentication failed. bool authenticateUser( HTTPServerRequest & request, diff --git a/src/Server/MySQLHandler.cpp b/src/Server/MySQLHandler.cpp index f2ac1184640..93f4bff46c2 100644 --- a/src/Server/MySQLHandler.cpp +++ b/src/Server/MySQLHandler.cpp @@ -3,11 +3,11 @@ #include #include #include -#include #include #include #include #include +#include #include #include #include @@ -19,9 +19,8 @@ #include #include #include -#include -#include #include +#include #if !defined(ARCADIA_BUILD) # include @@ -88,12 +87,10 @@ void MySQLHandler::run() setThreadName("MySQLHandler"); ThreadStatus thread_status; - session = std::make_shared(server.context(), ClientInfo::Interface::MYSQL, "MySQLWire"); - auto & session_client_info = session->getClientInfo(); + session = std::make_unique(server.context(), ClientInfo::Interface::MYSQL); + SCOPE_EXIT({ session.reset(); }); - session_client_info.current_address = socket().peerAddress(); - session_client_info.connection_id = connection_id; - session_client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + session->getClientInfo().connection_id = connection_id; in = std::make_shared(socket()); out = std::make_shared(socket()); @@ -127,12 +124,12 @@ void MySQLHandler::run() authenticate(handshake_response.username, handshake_response.auth_plugin_name, handshake_response.auth_response); - session_client_info.initial_user = handshake_response.username; - try { + session->makeSessionContext(); + session->sessionContext()->setDefaultFormat("MySQLWire"); if (!handshake_response.database.empty()) - session->setCurrentDatabase(handshake_response.database); + session->sessionContext()->setCurrentDatabase(handshake_response.database); } catch (const Exception & exc) { @@ -246,26 +243,23 @@ void MySQLHandler::finishHandshake(MySQLProtocol::ConnectionPhase::HandshakeResp void MySQLHandler::authenticate(const String & user_name, const String & auth_plugin_name, const String & initial_auth_response) { - // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used. - DB::Authentication::Type user_auth_type; try { - user_auth_type = session->getUserAuthentication(user_name).getType(); + // For compatibility with JavaScript MySQL client, Native41 authentication plugin is used when possible (if password is specified using double SHA1). Otherwise SHA256 plugin is used. + if (session->getAuthenticationType(user_name) == DB::Authentication::SHA256_PASSWORD) + { + authPluginSSL(); + } + + std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; + auth_plugin->authenticate(user_name, *session, auth_response, packet_endpoint, secure_connection, socket().peerAddress()); } - catch (const std::exception & e) + catch (const Exception & exc) { - session->onLogInFailure(user_name, e); + LOG_ERROR(log, "Authentication for user {} failed.", user_name); + packet_endpoint->sendPacket(ERRPacket(exc.code(), "00000", exc.message()), true); throw; } - - if (user_auth_type == DB::Authentication::SHA256_PASSWORD) - { - authPluginSSL(); - } - - std::optional auth_response = auth_plugin_name == auth_plugin->getName() ? std::make_optional(initial_auth_response) : std::nullopt; - auth_plugin->authenticate(user_name, auth_response, *session, packet_endpoint, secure_connection, socket().peerAddress()); - LOG_DEBUG(log, "Authentication for user {} succeeded.", user_name); } @@ -274,7 +268,7 @@ void MySQLHandler::comInitDB(ReadBuffer & payload) String database; readStringUntilEOF(database, payload); LOG_DEBUG(log, "Setting current database to {}", database); - session->setCurrentDatabase(database); + session->sessionContext()->setCurrentDatabase(database); packet_endpoint->sendPacket(OKPacket(0, client_capabilities, 0, 0, 1), true); } @@ -331,7 +325,9 @@ void MySQLHandler::comQuery(ReadBuffer & payload) ReadBufferFromString replacement(replacement_query); - auto query_context = session->makeQueryContext(Poco::format("mysql:%lu", connection_id)); + auto query_context = session->makeQueryContext(); + query_context->setCurrentQueryId(Poco::format("mysql:%lu", connection_id)); + CurrentThread::QueryScope query_scope{query_context}; std::atomic affected_rows {0}; auto prev = query_context->getProgressCallback(); @@ -343,8 +339,6 @@ void MySQLHandler::comQuery(ReadBuffer & payload) affected_rows += progress.written_rows; }); - CurrentThread::QueryScope query_scope{query_context}; - FormatSettings format_settings; format_settings.mysql_wire.client_capabilities = client_capabilities; format_settings.mysql_wire.max_packet_size = max_packet_size; diff --git a/src/Server/MySQLHandler.h b/src/Server/MySQLHandler.h index c57cb7d8f65..5258862cf23 100644 --- a/src/Server/MySQLHandler.h +++ b/src/Server/MySQLHandler.h @@ -63,7 +63,7 @@ protected: uint8_t sequence_id = 0; MySQLProtocol::PacketEndpointPtr packet_endpoint; - std::shared_ptr session; + std::unique_ptr session; using ReplacementFn = std::function; using Replacements = std::unordered_map; diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index ae21d387e73..0716d828520 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -2,8 +2,8 @@ #include #include #include +#include #include -#include #include "PostgreSQLHandler.h" #include #include @@ -53,14 +53,12 @@ void PostgreSQLHandler::run() setThreadName("PostgresHandler"); ThreadStatus thread_status; - Session session(server.context(), ClientInfo::Interface::POSTGRESQL, "PostgreSQLWire"); - auto & session_client_info = session.getClientInfo(); - - session_client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; + session = std::make_unique(server.context(), ClientInfo::Interface::POSTGRESQL); + SCOPE_EXIT({ session.reset(); }); try { - if (!startup(session)) + if (!startup()) return; while (true) @@ -71,7 +69,7 @@ void PostgreSQLHandler::run() switch (message_type) { case PostgreSQLProtocol::Messaging::FrontMessageType::QUERY: - processQuery(session); + processQuery(); break; case PostgreSQLProtocol::Messaging::FrontMessageType::TERMINATE: LOG_DEBUG(log, "Client closed the connection"); @@ -110,7 +108,7 @@ void PostgreSQLHandler::run() } -bool PostgreSQLHandler::startup(Session & session) +bool PostgreSQLHandler::startup() { Int32 payload_size; Int32 info; @@ -119,17 +117,20 @@ bool PostgreSQLHandler::startup(Session & session) if (static_cast(info) == PostgreSQLProtocol::Messaging::FrontMessageType::CANCEL_REQUEST) { LOG_DEBUG(log, "Client issued request canceling"); - cancelRequest(session); + cancelRequest(); return false; } std::unique_ptr start_up_msg = receiveStartupMessage(payload_size); - authentication_manager.authenticate(start_up_msg->user, session, *message_transport, socket().peerAddress()); + const auto & user_name = start_up_msg->user; + authentication_manager.authenticate(user_name, *session, *message_transport, socket().peerAddress()); try { + session->makeSessionContext(); + session->sessionContext()->setDefaultFormat("PostgreSQLWire"); if (!start_up_msg->database.empty()) - session.setCurrentDatabase(start_up_msg->database); + session->sessionContext()->setCurrentDatabase(start_up_msg->database); } catch (const Exception & exc) { @@ -207,18 +208,16 @@ void PostgreSQLHandler::sendParameterStatusData(PostgreSQLProtocol::Messaging::S message_transport->flush(); } -void PostgreSQLHandler::cancelRequest(Session & session) +void PostgreSQLHandler::cancelRequest() { - // TODO (nemkov): maybe run cancellation query with session context? - auto query_context = session.makeQueryContext(std::string{}); - query_context->setDefaultFormat("Null"); - std::unique_ptr msg = message_transport->receiveWithPayloadSize(8); String query = Poco::format("KILL QUERY WHERE query_id = 'postgres:%d:%d'", msg->process_id, msg->secret_key); ReadBufferFromString replacement(query); + auto query_context = session->makeQueryContext(); + query_context->setCurrentQueryId(""); executeQuery(replacement, *out, true, query_context, {}); } @@ -242,7 +241,7 @@ inline std::unique_ptr PostgreSQL return message; } -void PostgreSQLHandler::processQuery(Session & session) +void PostgreSQLHandler::processQuery() { try { @@ -265,7 +264,7 @@ void PostgreSQLHandler::processQuery(Session & session) return; } - const auto & settings = session.getSettings(); + const auto & settings = session->sessionContext()->getSettingsRef(); std::vector queries; auto parse_res = splitMultipartQuery(query->query, queries, settings.max_query_size, settings.max_parser_depth); if (!parse_res.second) @@ -278,7 +277,8 @@ void PostgreSQLHandler::processQuery(Session & session) for (const auto & spl_query : queries) { secret_key = dis(gen); - auto query_context = session.makeQueryContext(Poco::format("postgres:%d:%d", connection_id, secret_key)); + auto query_context = session->makeQueryContext(); + query_context->setCurrentQueryId(Poco::format("postgres:%d:%d", connection_id, secret_key)); CurrentThread::QueryScope query_scope{query_context}; ReadBufferFromString read_buf(spl_query); diff --git a/src/Server/PostgreSQLHandler.h b/src/Server/PostgreSQLHandler.h index cf4a6620063..36dd62d3dec 100644 --- a/src/Server/PostgreSQLHandler.h +++ b/src/Server/PostgreSQLHandler.h @@ -39,6 +39,7 @@ private: Poco::Logger * log = &Poco::Logger::get("PostgreSQLHandler"); IServer & server; + std::unique_ptr session; bool ssl_enabled = false; Int32 connection_id = 0; Int32 secret_key = 0; @@ -57,7 +58,7 @@ private: void changeIO(Poco::Net::StreamSocket & socket); - bool startup(Session & session); + bool startup(); void establishSecureConnection(Int32 & payload_size, Int32 & info); @@ -65,11 +66,11 @@ private: void sendParameterStatusData(PostgreSQLProtocol::Messaging::StartupMessage & start_up_message); - void cancelRequest(Session & session); + void cancelRequest(); std::unique_ptr receiveStartupMessage(int payload_size); - void processQuery(DB::Session & session); + void processQuery(); static bool isEmptyQuery(const String & query); }; diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index de14f117981..b2db65e22bc 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -20,16 +20,16 @@ #include #include #include -#include #include #include #include -#include #include +#include #include #include #include #include +#include #include #include #include @@ -75,7 +75,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int ATTEMPT_TO_READ_AFTER_EOF; extern const int CLIENT_HAS_CONNECTED_TO_WRONG_PORT; - extern const int UNKNOWN_DATABASE; extern const int UNKNOWN_EXCEPTION; extern const int UNKNOWN_PACKET_FROM_CLIENT; extern const int POCO_EXCEPTION; @@ -90,7 +89,6 @@ TCPHandler::TCPHandler(IServer & server_, const Poco::Net::StreamSocket & socket , server(server_) , parse_proxy_protocol(parse_proxy_protocol_) , log(&Poco::Logger::get("TCPHandler")) - , query_context(Context::createCopy(server.context())) , server_display_name(std::move(server_display_name_)) { } @@ -115,16 +113,10 @@ void TCPHandler::runImpl() ThreadStatus thread_status; session = std::make_unique(server.context(), ClientInfo::Interface::TCP); - const auto session_context = session->sessionContext(); + extractConnectionSettingsFromContext(server.context()); - /// These timeouts can be changed after receiving query. - const auto & settings = session->getSettings(); - - auto global_receive_timeout = settings.receive_timeout; - auto global_send_timeout = settings.send_timeout; - - socket().setReceiveTimeout(global_receive_timeout); - socket().setSendTimeout(global_send_timeout); + socket().setReceiveTimeout(receive_timeout); + socket().setSendTimeout(send_timeout); socket().setNoDelay(true); in = std::make_shared(socket()); @@ -162,33 +154,27 @@ void TCPHandler::runImpl() try { /// We try to send error information to the client. - sendException(e, session->getSettings().calculate_text_stack_trace); + sendException(e, send_exception_with_stack_trace); } catch (...) {} throw; } - /// When connecting, the default database can be specified. - if (!default_database.empty()) - { - if (!DatabaseCatalog::instance().isDatabaseExist(default_database)) - { - Exception e("Database " + backQuote(default_database) + " doesn't exist", ErrorCodes::UNKNOWN_DATABASE); - LOG_ERROR(log, getExceptionMessage(e, true)); - sendException(e, settings.calculate_text_stack_trace); - return; - } - - session->setCurrentDatabase(default_database); - } - - UInt64 idle_connection_timeout = settings.idle_connection_timeout; - UInt64 poll_interval = settings.poll_interval; - sendHello(); - session->mutableSessionContext()->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); + if (!is_interserver_mode) /// In interserver mode queries are executed without a session context. + { + session->makeSessionContext(); + + /// If session created, then settings in session context has been updated. + /// So it's better to update the connection settings for flexibility. + extractConnectionSettingsFromContext(session->sessionContext()); + + /// When connecting, the default database could be specified. + if (!default_database.empty()) + session->sessionContext()->setCurrentDatabase(default_database); + } while (true) { @@ -210,10 +196,6 @@ void TCPHandler::runImpl() if (server.isCancelled() || in->eof()) break; - /// Set context of request. - /// TODO (nemkov): create query later in receiveQuery - query_context = session->makeQueryContext(std::string{}); // proper query_id is set later in receiveQuery - Stopwatch watch; state.reset(); @@ -226,8 +208,6 @@ void TCPHandler::runImpl() std::optional exception; bool network_error = false; - bool send_exception_with_stack_trace = true; - try { /// If a user passed query-local timeouts, reset socket to initial state at the end of the query @@ -240,23 +220,22 @@ void TCPHandler::runImpl() if (!receivePacket()) continue; - /** If Query received, then settings in query_context has been updated - * So, update some other connection settings, for flexibility. - */ - { - const Settings & query_settings = query_context->getSettingsRef(); - idle_connection_timeout = query_settings.idle_connection_timeout; - poll_interval = query_settings.poll_interval; - } - /** If part_uuids got received in previous packet, trying to read again. */ - if (state.empty() && state.part_uuids && !receivePacket()) + if (state.empty() && state.part_uuids_to_ignore && !receivePacket()) continue; query_scope.emplace(query_context); - send_exception_with_stack_trace = query_context->getSettingsRef().calculate_text_stack_trace; + /// If query received, then settings in query_context has been updated. + /// So it's better to update the connection settings for flexibility. + extractConnectionSettingsFromContext(query_context); + + /// Sync timeouts on client and server during current query to avoid dangling queries on server + /// NOTE: We use send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), + /// because send_timeout is client-side setting which has opposite meaning on the server side. + /// NOTE: these settings are applied only for current connection (not for distributed tables' connections) + state.timeout_setter = std::make_unique(socket(), receive_timeout, send_timeout); /// Should we send internal logs to client? const auto client_logs_level = query_context->getSettingsRef().send_logs_level; @@ -269,20 +248,18 @@ void TCPHandler::runImpl() CurrentThread::setFatalErrorCallback([this]{ sendLogs(); }); } - query_context->setExternalTablesInitializer([&settings, this] (ContextPtr context) + query_context->setExternalTablesInitializer([this] (ContextPtr context) { if (context != query_context) throw Exception("Unexpected context in external tables initializer", ErrorCodes::LOGICAL_ERROR); /// Get blocks of temporary tables - readData(settings); + readData(); /// Reset the input stream, as we received an empty block while receiving external table data. /// So, the stream has been marked as cancelled and we can't read from it anymore. state.block_in.reset(); state.maybe_compressed_in.reset(); /// For more accurate accounting by MemoryTracker. - - state.temporary_tables_read = true; }); /// Send structure of columns to client for function input() @@ -306,15 +283,12 @@ void TCPHandler::runImpl() sendData(state.input_header); }); - query_context->setInputBlocksReaderCallback([&settings, this] (ContextPtr context) -> Block + query_context->setInputBlocksReaderCallback([this] (ContextPtr context) -> Block { if (context != query_context) throw Exception("Unexpected context in InputBlocksReader", ErrorCodes::LOGICAL_ERROR); - size_t poll_interval_ms; - int receive_timeout; - std::tie(poll_interval_ms, receive_timeout) = getReadTimeouts(settings); - if (!readDataNext(poll_interval_ms, receive_timeout)) + if (!readDataNext()) { state.block_in.reset(); state.maybe_compressed_in.reset(); @@ -337,15 +311,13 @@ void TCPHandler::runImpl() /// Processing Query state.io = executeQuery(state.query, query_context, false, state.stage, may_have_embedded_data); - unknown_packet_in_send_data = query_context->getSettingsRef().unknown_packet_in_send_data; - after_check_cancelled.restart(); after_send_progress.restart(); if (state.io.out) { state.need_receive_data_for_insert = true; - processInsertQuery(settings); + processInsertQuery(); } else if (state.need_receive_data_for_input) // It implies pipeline execution { @@ -461,16 +433,17 @@ void TCPHandler::runImpl() try { - if (exception && !state.temporary_tables_read) - query_context->initializeExternalTablesIfSet(); + /// A query packet is always followed by one or more data packets. + /// If some of those data packets are left, try to skip them. + if (exception && !state.empty() && !state.read_all_data) + skipData(); } catch (...) { network_error = true; - LOG_WARNING(log, "Can't read external tables after query failure."); + LOG_WARNING(log, "Can't skip data packets after query failure."); } - try { /// QueryState should be cleared before QueryScope, since otherwise @@ -501,75 +474,94 @@ void TCPHandler::runImpl() } -bool TCPHandler::readDataNext(size_t poll_interval, time_t receive_timeout) +void TCPHandler::extractConnectionSettingsFromContext(const ContextPtr & context) +{ + const auto & settings = context->getSettingsRef(); + send_exception_with_stack_trace = settings.calculate_text_stack_trace; + send_timeout = settings.send_timeout; + receive_timeout = settings.receive_timeout; + poll_interval = settings.poll_interval; + idle_connection_timeout = settings.idle_connection_timeout; + interactive_delay = settings.interactive_delay; + sleep_in_send_tables_status = settings.sleep_in_send_tables_status_ms; + unknown_packet_in_send_data = settings.unknown_packet_in_send_data; + sleep_in_receive_cancel = settings.sleep_in_receive_cancel_ms; +} + + +bool TCPHandler::readDataNext() { Stopwatch watch(CLOCK_MONOTONIC_COARSE); + /// Poll interval should not be greater than receive_timeout + constexpr UInt64 min_timeout_ms = 5000; // 5 ms + UInt64 timeout_ms = std::max(min_timeout_ms, std::min(poll_interval * 1000000, static_cast(receive_timeout.totalMicroseconds()))); + bool read_ok = false; + /// We are waiting for a packet from the client. Thus, every `POLL_INTERVAL` seconds check whether we need to shut down. while (true) { - if (static_cast(*in).poll(poll_interval)) + if (static_cast(*in).poll(timeout_ms)) + { + /// If client disconnected. + if (in->eof()) + { + LOG_INFO(log, "Client has dropped the connection, cancel the query."); + state.is_connection_closed = true; + break; + } + + /// We accept and process data. + read_ok = receivePacket(); break; + } /// Do we need to shut down? if (server.isCancelled()) - return false; + break; /** Have we waited for data for too long? * If we periodically poll, the receive_timeout of the socket itself does not work. * Therefore, an additional check is added. */ Float64 elapsed = watch.elapsedSeconds(); - if (elapsed > static_cast(receive_timeout)) + if (elapsed > static_cast(receive_timeout.totalSeconds())) { throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Timeout exceeded while receiving data from client. Waited for {} seconds, timeout is {} seconds.", - static_cast(elapsed), receive_timeout); + static_cast(elapsed), receive_timeout.totalSeconds()); } } - /// If client disconnected. - if (in->eof()) - { - LOG_INFO(log, "Client has dropped the connection, cancel the query."); - state.is_connection_closed = true; - return false; - } + if (read_ok) + sendLogs(); + else + state.read_all_data = true; - /// We accept and process data. And if they are over, then we leave. - if (!receivePacket()) - return false; - - sendLogs(); - return true; + return read_ok; } -std::tuple TCPHandler::getReadTimeouts(const Settings & connection_settings) +void TCPHandler::readData() { - const auto receive_timeout = query_context->getSettingsRef().receive_timeout.value; - - /// Poll interval should not be greater than receive_timeout - const size_t default_poll_interval = connection_settings.poll_interval * 1000000; - size_t current_poll_interval = static_cast(receive_timeout.totalMicroseconds()); - constexpr size_t min_poll_interval = 5000; // 5 ms - size_t poll_interval = std::max(min_poll_interval, std::min(default_poll_interval, current_poll_interval)); - - return std::make_tuple(poll_interval, receive_timeout.totalSeconds()); -} - - -void TCPHandler::readData(const Settings & connection_settings) -{ - auto [poll_interval, receive_timeout] = getReadTimeouts(connection_settings); sendLogs(); - while (readDataNext(poll_interval, receive_timeout)) + while (readDataNext()) ; } -void TCPHandler::processInsertQuery(const Settings & connection_settings) +void TCPHandler::skipData() +{ + state.skipping_data = true; + SCOPE_EXIT({ state.skipping_data = false; }); + + while (readDataNext()) + ; +} + + +void TCPHandler::processInsertQuery() { /** Made above the rest of the lines, so that in case of `writePrefix` function throws an exception, * client receive exception before sending data. @@ -595,7 +587,7 @@ void TCPHandler::processInsertQuery(const Settings & connection_settings) try { - readData(connection_settings); + readData(); } catch (...) { @@ -634,7 +626,7 @@ void TCPHandler::processOrdinaryQuery() break; } - if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) + if (after_send_progress.elapsed() / 1000 >= interactive_delay) { /// Some time passed. after_send_progress.restart(); @@ -643,7 +635,7 @@ void TCPHandler::processOrdinaryQuery() sendLogs(); - if (async_in.poll(query_context->getSettingsRef().interactive_delay / 1000)) + if (async_in.poll(interactive_delay / 1000)) { const auto block = async_in.read(); if (!block) @@ -698,7 +690,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() CurrentMetrics::Increment query_thread_metric_increment{CurrentMetrics::QueryThread}; Block block; - while (executor.pull(block, query_context->getSettingsRef().interactive_delay / 1000)) + while (executor.pull(block, interactive_delay / 1000)) { std::lock_guard lock(task_callback_mutex); @@ -709,7 +701,7 @@ void TCPHandler::processOrdinaryQueryWithProcessors() break; } - if (after_send_progress.elapsed() / 1000 >= query_context->getSettingsRef().interactive_delay) + if (after_send_progress.elapsed() / 1000 >= interactive_delay) { /// Some time passed and there is a progress. after_send_progress.restart(); @@ -755,13 +747,14 @@ void TCPHandler::processTablesStatusRequest() { TablesStatusRequest request; request.read(*in, client_tcp_protocol_version); - const auto session_context = session->sessionContext(); + + ContextPtr context_to_resolve_table_names = session->sessionContext() ? session->sessionContext() : server.context(); TablesStatusResponse response; for (const QualifiedTableName & table_name: request.tables) { - auto resolved_id = session_context->tryResolveStorageID({table_name.database, table_name.table}); - StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, session_context); + auto resolved_id = context_to_resolve_table_names->tryResolveStorageID({table_name.database, table_name.table}); + StoragePtr table = DatabaseCatalog::instance().tryGetTable(resolved_id, context_to_resolve_table_names); if (!table) continue; @@ -781,11 +774,10 @@ void TCPHandler::processTablesStatusRequest() writeVarUInt(Protocol::Server::TablesStatusResponse, *out); /// For testing hedged requests - const Settings & settings = query_context->getSettingsRef(); - if (settings.sleep_in_send_tables_status_ms.totalMilliseconds()) + if (sleep_in_send_tables_status.totalMilliseconds()) { out->next(); - std::chrono::milliseconds ms(settings.sleep_in_send_tables_status_ms.totalMilliseconds()); + std::chrono::milliseconds ms(sleep_in_send_tables_status.totalMilliseconds()); std::this_thread::sleep_for(ms); } @@ -977,22 +969,21 @@ void TCPHandler::receiveHello() (!user.empty() ? ", user: " + user : "") ); - if (user != USER_INTERSERVER_MARKER) - { - auto & client_info = session->getClientInfo(); - client_info.interface = ClientInfo::Interface::TCP; - client_info.client_name = client_name; - client_info.client_version_major = client_version_major; - client_info.client_version_minor = client_version_minor; - client_info.client_version_patch = client_version_patch; - client_info.client_tcp_protocol_version = client_tcp_protocol_version; + auto & client_info = session->getClientInfo(); + client_info.client_name = client_name; + client_info.client_version_major = client_version_major; + client_info.client_version_minor = client_version_minor; + client_info.client_version_patch = client_version_patch; + client_info.client_tcp_protocol_version = client_tcp_protocol_version; - session->setUser(user, password, socket().peerAddress()); - } - else + is_interserver_mode = (user == USER_INTERSERVER_MARKER); + if (is_interserver_mode) { receiveClusterNameAndSalt(); + return; } + + session->authenticate(user, password, socket().peerAddress()); } @@ -1039,8 +1030,11 @@ bool TCPHandler::receivePacket() { case Protocol::Client::IgnoredPartUUIDs: /// Part uuids packet if any comes before query. + if (!state.empty() || state.part_uuids_to_ignore) + receiveUnexpectedIgnoredPartUUIDs(); receiveIgnoredPartUUIDs(); return true; + case Protocol::Client::Query: if (!state.empty()) receiveUnexpectedQuery(); @@ -1049,8 +1043,10 @@ bool TCPHandler::receivePacket() case Protocol::Client::Data: case Protocol::Client::Scalar: + if (state.skipping_data) + return receiveUnexpectedData(false); if (state.empty()) - receiveUnexpectedData(); + receiveUnexpectedData(true); return receiveData(packet_type == Protocol::Client::Scalar); case Protocol::Client::Ping: @@ -1061,10 +1057,9 @@ bool TCPHandler::receivePacket() case Protocol::Client::Cancel: { /// For testing connection collector. - const Settings & settings = query_context->getSettingsRef(); - if (settings.sleep_in_receive_cancel_ms.totalMilliseconds()) + if (sleep_in_receive_cancel.totalMilliseconds()) { - std::chrono::milliseconds ms(settings.sleep_in_receive_cancel_ms.totalMilliseconds()); + std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); std::this_thread::sleep_for(ms); } @@ -1086,14 +1081,18 @@ bool TCPHandler::receivePacket() } } + void TCPHandler::receiveIgnoredPartUUIDs() { - state.part_uuids = true; - std::vector uuids; - readVectorBinary(uuids, *in); + readVectorBinary(state.part_uuids_to_ignore.emplace(), *in); +} - if (!uuids.empty()) - query_context->getIgnoredPartUUIDs()->add(uuids); + +void TCPHandler::receiveUnexpectedIgnoredPartUUIDs() +{ + std::vector skip_part_uuids; + readVectorBinary(skip_part_uuids, *in); + throw NetException("Unexpected packet IgnoredPartUUIDs received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); } @@ -1107,10 +1106,9 @@ String TCPHandler::receiveReadTaskResponseAssumeLocked() { state.is_cancelled = true; /// For testing connection collector. - const Settings & settings = query_context->getSettingsRef(); - if (settings.sleep_in_receive_cancel_ms.totalMilliseconds()) + if (sleep_in_receive_cancel.totalMilliseconds()) { - std::chrono::milliseconds ms(settings.sleep_in_receive_cancel_ms.totalMilliseconds()); + std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); std::this_thread::sleep_for(ms); } return {}; @@ -1141,14 +1139,14 @@ void TCPHandler::receiveClusterNameAndSalt() if (salt.empty()) throw NetException("Empty salt is not allowed", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); - cluster_secret = query_context->getCluster(cluster)->getSecret(); + cluster_secret = server.context()->getCluster(cluster)->getSecret(); } catch (const Exception & e) { try { /// We try to send error information to the client. - sendException(e, session->getSettings().calculate_text_stack_trace); + sendException(e, send_exception_with_stack_trace); } catch (...) {} @@ -1163,27 +1161,12 @@ void TCPHandler::receiveQuery() state.is_empty = false; readStringBinary(state.query_id, *in); -// query_context = session->makeQueryContext(state.query_id); - /// Client info - ClientInfo & client_info = query_context->getClientInfo(); + /// Read client info. + ClientInfo client_info = session->getClientInfo(); if (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_CLIENT_INFO) client_info.read(*in, client_tcp_protocol_version); - /// For better support of old clients, that does not send ClientInfo. - if (client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) - { - client_info.query_kind = ClientInfo::QueryKind::INITIAL_QUERY; - client_info.client_name = client_name; - client_info.client_version_major = client_version_major; - client_info.client_version_minor = client_version_minor; - client_info.client_version_patch = client_version_patch; - client_info.client_tcp_protocol_version = client_tcp_protocol_version; - } - - /// Set fields, that are known apriori. - client_info.interface = ClientInfo::Interface::TCP; - /// Per query settings are also passed via TCP. /// We need to check them before applying due to they can violate the settings constraints. auto settings_format = (client_tcp_protocol_version >= DBMS_MIN_REVISION_WITH_SETTINGS_SERIALIZED_AS_STRINGS) @@ -1204,12 +1187,11 @@ void TCPHandler::receiveQuery() readVarUInt(compression, *in); state.compression = static_cast(compression); + last_block_in.compression = state.compression; readStringBinary(state.query, *in); - /// It is OK to check only when query != INITIAL_QUERY, - /// since only in that case the actions will be done. - if (!cluster.empty() && client_info.query_kind != ClientInfo::QueryKind::INITIAL_QUERY) + if (is_interserver_mode) { #if USE_SSL std::string data(salt); @@ -1231,26 +1213,33 @@ void TCPHandler::receiveQuery() /// i.e. when the INSERT is done with the global context (w/o user). if (!client_info.initial_user.empty()) { - query_context->setUserWithoutCheckingPassword(client_info.initial_user, client_info.initial_address); - LOG_DEBUG(log, "User (initial): {}", query_context->getUserName()); + LOG_DEBUG(log, "User (initial): {}", client_info.initial_user); + session->authenticate(AlwaysAllowCredentials{client_info.initial_user}, client_info.initial_address); } - /// No need to update connection_context, since it does not requires user (it will not be used for query execution) #else throw Exception( "Inter-server secret support is disabled, because ClickHouse was built without SSL library", ErrorCodes::SUPPORT_IS_DISABLED); #endif } - else - { - query_context->setInitialRowPolicy(); - } + + query_context = session->makeQueryContext(std::move(client_info)); + + /// Sets the default database if it wasn't set earlier for the session context. + if (!default_database.empty() && !session->sessionContext()) + query_context->setCurrentDatabase(default_database); + + if (state.part_uuids_to_ignore) + query_context->getIgnoredPartUUIDs()->add(*state.part_uuids_to_ignore); + + query_context->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); /// /// Settings /// auto settings_changes = passed_settings.changes(); - if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) + auto query_kind = query_context->getClientInfo().query_kind; + if (query_kind == ClientInfo::QueryKind::INITIAL_QUERY) { /// Throw an exception if the passed settings violate the constraints. query_context->checkSettingsConstraints(settings_changes); @@ -1262,40 +1251,24 @@ void TCPHandler::receiveQuery() } query_context->applySettingsChanges(settings_changes); + /// Use the received query id, or generate a random default. It is convenient + /// to also generate the default OpenTelemetry trace id at the same time, and + /// set the trace parent. + /// Notes: + /// 1) ClientInfo might contain upstream trace id, so we decide whether to use + /// the default ids after we have received the ClientInfo. + /// 2) There is the opentelemetry_start_trace_probability setting that + /// controls when we start a new trace. It can be changed via Native protocol, + /// so we have to apply the changes first. + query_context->setCurrentQueryId(state.query_id); + /// Disable function name normalization when it's a secondary query, because queries are either /// already normalized on initiator node, or not normalized and should remain unnormalized for /// compatibility. - if (client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) + if (query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) { query_context->setSetting("normalize_function_names", Field(0)); } - - // Use the received query id, or generate a random default. It is convenient - // to also generate the default OpenTelemetry trace id at the same time, and - // set the trace parent. - // Why is this done here and not earlier: - // 1) ClientInfo might contain upstream trace id, so we decide whether to use - // the default ids after we have received the ClientInfo. - // 2) There is the opentelemetry_start_trace_probability setting that - // controls when we start a new trace. It can be changed via Native protocol, - // so we have to apply the changes first. - query_context->setCurrentQueryId(state.query_id); - - // Set parameters of initial query. - if (client_info.query_kind == ClientInfo::QueryKind::INITIAL_QUERY) - { - /// 'Current' fields was set at receiveHello. - client_info.initial_user = client_info.current_user; - client_info.initial_query_id = client_info.current_query_id; - client_info.initial_address = client_info.current_address; - } - - /// Sync timeouts on client and server during current query to avoid dangling queries on server - /// NOTE: We use settings.send_timeout for the receive timeout and vice versa (change arguments ordering in TimeoutSetter), - /// because settings.send_timeout is client-side setting which has opposite meaning on the server side. - /// NOTE: these settings are applied only for current connection (not for distributed tables' connections) - const Settings & settings = query_context->getSettingsRef(); - state.timeout_setter = std::make_unique(socket(), settings.receive_timeout, settings.send_timeout); } void TCPHandler::receiveUnexpectedQuery() @@ -1320,7 +1293,10 @@ void TCPHandler::receiveUnexpectedQuery() readStringBinary(skip_hash, *in, 32); readVarUInt(skip_uint_64, *in); + readVarUInt(skip_uint_64, *in); + last_block_in.compression = static_cast(skip_uint_64); + readStringBinary(skip_string, *in); throw NetException("Unexpected packet Query received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); @@ -1337,73 +1313,77 @@ bool TCPHandler::receiveData(bool scalar) /// Read one block from the network and write it down Block block = state.block_in->read(); - if (block) + if (!block) { - if (scalar) - { - /// Scalar value - query_context->addScalar(temporary_id.table_name, block); - } - else if (!state.need_receive_data_for_insert && !state.need_receive_data_for_input) - { - /// Data for external tables + state.read_all_data = true; + return false; + } - auto resolved = query_context->tryResolveStorageID(temporary_id, Context::ResolveExternal); - StoragePtr storage; - /// If such a table does not exist, create it. - if (resolved) - { - storage = DatabaseCatalog::instance().getTable(resolved, query_context); - } - else - { - NamesAndTypesList columns = block.getNamesAndTypesList(); - auto temporary_table = TemporaryTableHolder(query_context, ColumnsDescription{columns}, {}); - storage = temporary_table.getTable(); - query_context->addExternalTable(temporary_id.table_name, std::move(temporary_table)); - } - auto metadata_snapshot = storage->getInMemoryMetadataPtr(); - /// The data will be written directly to the table. - auto temporary_table_out = std::make_shared(storage->write(ASTPtr(), metadata_snapshot, query_context)); - temporary_table_out->write(block); - temporary_table_out->writeSuffix(); + if (scalar) + { + /// Scalar value + query_context->addScalar(temporary_id.table_name, block); + } + else if (!state.need_receive_data_for_insert && !state.need_receive_data_for_input) + { + /// Data for external tables - } - else if (state.need_receive_data_for_input) + auto resolved = query_context->tryResolveStorageID(temporary_id, Context::ResolveExternal); + StoragePtr storage; + /// If such a table does not exist, create it. + if (resolved) { - /// 'input' table function. - state.block_for_input = block; + storage = DatabaseCatalog::instance().getTable(resolved, query_context); } else { - /// INSERT query. - state.io.out->write(block); + NamesAndTypesList columns = block.getNamesAndTypesList(); + auto temporary_table = TemporaryTableHolder(query_context, ColumnsDescription{columns}, {}); + storage = temporary_table.getTable(); + query_context->addExternalTable(temporary_id.table_name, std::move(temporary_table)); } - return true; + auto metadata_snapshot = storage->getInMemoryMetadataPtr(); + /// The data will be written directly to the table. + auto temporary_table_out = std::make_shared(storage->write(ASTPtr(), metadata_snapshot, query_context)); + temporary_table_out->write(block); + temporary_table_out->writeSuffix(); + + } + else if (state.need_receive_data_for_input) + { + /// 'input' table function. + state.block_for_input = block; } else - return false; + { + /// INSERT query. + state.io.out->write(block); + } + return true; } -void TCPHandler::receiveUnexpectedData() + +bool TCPHandler::receiveUnexpectedData(bool throw_exception) { String skip_external_table_name; readStringBinary(skip_external_table_name, *in); std::shared_ptr maybe_compressed_in; - if (last_block_in.compression == Protocol::Compression::Enable) maybe_compressed_in = std::make_shared(*in, /* allow_different_codecs */ true); else maybe_compressed_in = in; - auto skip_block_in = std::make_shared( - *maybe_compressed_in, - last_block_in.header, - client_tcp_protocol_version); + auto skip_block_in = std::make_shared(*maybe_compressed_in, client_tcp_protocol_version); + bool read_ok = skip_block_in->read(); - skip_block_in->read(); - throw NetException("Unexpected packet Data received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); + if (!read_ok) + state.read_all_data = true; + + if (throw_exception) + throw NetException("Unexpected packet Data received from client", ErrorCodes::UNEXPECTED_PACKET_FROM_CLIENT); + + return read_ok; } void TCPHandler::initBlockInput() @@ -1424,9 +1404,6 @@ void TCPHandler::initBlockInput() else if (state.need_receive_data_for_input) header = state.input_header; - last_block_in.header = header; - last_block_in.compression = state.compression; - state.block_in = std::make_shared( *state.maybe_compressed_in, header, @@ -1439,10 +1416,9 @@ void TCPHandler::initBlockOutput(const Block & block) { if (!state.block_out) { + const Settings & query_settings = query_context->getSettingsRef(); if (!state.maybe_compressed_out) { - const Settings & query_settings = query_context->getSettingsRef(); - std::string method = Poco::toUpper(query_settings.network_compression_method.toString()); std::optional level; if (method == "ZSTD") @@ -1463,7 +1439,7 @@ void TCPHandler::initBlockOutput(const Block & block) *state.maybe_compressed_out, client_tcp_protocol_version, block.cloneEmpty(), - !session->getSettings().low_cardinality_allow_in_native_format); + !query_settings.low_cardinality_allow_in_native_format); } } @@ -1472,11 +1448,12 @@ void TCPHandler::initLogsBlockOutput(const Block & block) if (!state.logs_block_out) { /// Use uncompressed stream since log blocks usually contain only one row + const Settings & query_settings = query_context->getSettingsRef(); state.logs_block_out = std::make_shared( *out, client_tcp_protocol_version, block.cloneEmpty(), - !session->getSettings().low_cardinality_allow_in_native_format); + !query_settings.low_cardinality_allow_in_native_format); } } @@ -1486,7 +1463,7 @@ bool TCPHandler::isQueryCancelled() if (state.is_cancelled || state.sent_all_data) return true; - if (after_check_cancelled.elapsed() / 1000 < query_context->getSettingsRef().interactive_delay) + if (after_check_cancelled.elapsed() / 1000 < interactive_delay) return false; after_check_cancelled.restart(); @@ -1514,10 +1491,9 @@ bool TCPHandler::isQueryCancelled() state.is_cancelled = true; /// For testing connection collector. { - const Settings & settings = query_context->getSettingsRef(); - if (settings.sleep_in_receive_cancel_ms.totalMilliseconds()) + if (sleep_in_receive_cancel.totalMilliseconds()) { - std::chrono::milliseconds ms(settings.sleep_in_receive_cancel_ms.totalMilliseconds()); + std::chrono::milliseconds ms(sleep_in_receive_cancel.totalMilliseconds()); std::this_thread::sleep_for(ms); } } @@ -1555,11 +1531,10 @@ void TCPHandler::sendData(const Block & block) writeStringBinary("", *out); /// For testing hedged requests - const Settings & settings = query_context->getSettingsRef(); - if (block.rows() > 0 && settings.sleep_in_send_data_ms.totalMilliseconds()) + if (block.rows() > 0 && query_context->getSettingsRef().sleep_in_send_data_ms.totalMilliseconds()) { out->next(); - std::chrono::milliseconds ms(settings.sleep_in_send_data_ms.totalMilliseconds()); + std::chrono::milliseconds ms(query_context->getSettingsRef().sleep_in_send_data_ms.totalMilliseconds()); std::this_thread::sleep_for(ms); } diff --git a/src/Server/TCPHandler.h b/src/Server/TCPHandler.h index d8e156ee7be..7f75d0ac04b 100644 --- a/src/Server/TCPHandler.h +++ b/src/Server/TCPHandler.h @@ -27,7 +27,9 @@ namespace DB { class Session; +struct Settings; class ColumnsDescription; +struct BlockStreamProfileInfo; /// State of query processing. struct QueryState @@ -66,11 +68,11 @@ struct QueryState bool sent_all_data = false; /// Request requires data from the client (INSERT, but not INSERT SELECT). bool need_receive_data_for_insert = false; - /// Temporary tables read - bool temporary_tables_read = false; + /// Data was read. + bool read_all_data = false; /// A state got uuids to exclude from a query - bool part_uuids = false; + std::optional> part_uuids_to_ignore; /// Request requires data from client for function input() bool need_receive_data_for_input = false; @@ -79,6 +81,9 @@ struct QueryState /// sample block from StorageInput Block input_header; + /// If true, the data packets will be skipped instead of reading. Used to recover after errors. + bool skipping_data = false; + /// To output progress, the difference after the previous sending of progress. Progress progress; @@ -100,7 +105,6 @@ struct QueryState struct LastBlockInputParameters { Protocol::Compression compression = Protocol::Compression::Disable; - Block header; }; class TCPHandler : public Poco::Net::TCPServerConnection @@ -133,11 +137,20 @@ private: UInt64 client_version_patch = 0; UInt64 client_tcp_protocol_version = 0; + /// Connection settings, which are extracted from a context. + bool send_exception_with_stack_trace = true; + Poco::Timespan send_timeout = DBMS_DEFAULT_SEND_TIMEOUT_SEC; + Poco::Timespan receive_timeout = DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC; + UInt64 poll_interval = DBMS_DEFAULT_POLL_INTERVAL; + UInt64 idle_connection_timeout = 3600; + UInt64 interactive_delay = 100000; + Poco::Timespan sleep_in_send_tables_status; + UInt64 unknown_packet_in_send_data = 0; + Poco::Timespan sleep_in_receive_cancel; + std::unique_ptr session; ContextMutablePtr query_context; - size_t unknown_packet_in_send_data = 0; - /// Streams for reading/writing from/to client connection socket. std::shared_ptr in; std::shared_ptr out; @@ -149,6 +162,7 @@ private: String default_database; /// For inter-server secret (remote_server.*.secret) + bool is_interserver_mode = false; String salt; String cluster; String cluster_secret; @@ -168,6 +182,8 @@ private: void runImpl(); + void extractConnectionSettingsFromContext(const ContextPtr & context); + bool receiveProxyHeader(); void receiveHello(); bool receivePacket(); @@ -175,18 +191,19 @@ private: void receiveIgnoredPartUUIDs(); String receiveReadTaskResponseAssumeLocked(); bool receiveData(bool scalar); - bool readDataNext(size_t poll_interval, time_t receive_timeout); - void readData(const Settings & connection_settings); + bool readDataNext(); + void readData(); + void skipData(); void receiveClusterNameAndSalt(); - std::tuple getReadTimeouts(const Settings & connection_settings); - [[noreturn]] void receiveUnexpectedData(); + bool receiveUnexpectedData(bool throw_exception = true); [[noreturn]] void receiveUnexpectedQuery(); + [[noreturn]] void receiveUnexpectedIgnoredPartUUIDs(); [[noreturn]] void receiveUnexpectedHello(); [[noreturn]] void receiveUnexpectedTablesStatusRequest(); /// Process INSERT query - void processInsertQuery(const Settings & connection_settings); + void processInsertQuery(); /// Process a request that does not require the receiving of data blocks from the client void processOrdinaryQuery(); diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index 92387b13d55..09f9cf8b1f5 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -61,9 +61,8 @@ void TableFunctionMySQL::parseArguments(const ASTPtr & ast_function, ContextPtr user_name = args[3]->as().value.safeGet(); password = args[4]->as().value.safeGet(); - const auto & settings = context->getSettingsRef(); /// Split into replicas if needed. 3306 is the default MySQL port number - const size_t max_addresses = settings.glob_expansion_max_elements; + size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; auto addresses = parseRemoteDescriptionForExternalDatabase(host_port, max_addresses, 3306); pool.emplace(remote_database_name, addresses, user_name, password); diff --git a/tests/integration/test_read_temporary_tables_on_failure/test.py b/tests/integration/test_read_temporary_tables_on_failure/test.py index e62c7c9eaec..ae59fb31641 100644 --- a/tests/integration/test_read_temporary_tables_on_failure/test.py +++ b/tests/integration/test_read_temporary_tables_on_failure/test.py @@ -24,3 +24,4 @@ def test_different_versions(start_cluster): node.query("SELECT 1", settings={'max_concurrent_queries_for_user': 1}) assert node.contains_in_log('Too many simultaneous queries for user') assert not node.contains_in_log('Unknown packet') + assert not node.contains_in_log('Unexpected packet') diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference index b40e4f87c13..f45f1ab6104 100644 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.reference +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.reference @@ -1,8 +1,20 @@ ===http=== +{"query":"select 1 from remote('127.0.0.2', system, one) format Null\n","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"DESC TABLE system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"SELECT 1 FROM system.one","status":"QueryFinish","tracestate":"some custom state","sorted_by_start_time":1} +{"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"DESC TABLE system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"SELECT 1 FROM system.one","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} +{"query":"select 1 from remote('127.0.0.2', system, one) format Null\n","query_status":"QueryFinish","tracestate":"some custom state","sorted_by_finish_time":1} {"total spans":"4","unique spans":"4","unique non-zero parent spans":"3"} {"initial query spans with proper parent":"1"} {"unique non-empty tracestate values":"1"} ===native=== +{"query":"select * from url('http:\/\/127.0.0.2:8123\/?query=select%201%20format%20Null', CSV, 'a int')","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} +{"query":"select 1 format Null\n","status":"QueryFinish","tracestate":"another custom state","sorted_by_start_time":1} +{"query":"select 1 format Null\n","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} +{"query":"select * from url('http:\/\/127.0.0.2:8123\/?query=select%201%20format%20Null', CSV, 'a int')","query_status":"QueryFinish","tracestate":"another custom state","sorted_by_finish_time":1} {"total spans":"2","unique spans":"2","unique non-zero parent spans":"2"} {"initial query spans with proper parent":"1"} {"unique non-empty tracestate values":"1"} diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index 8f034b0bf61..59cd1b57d1e 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -12,6 +12,28 @@ function check_log ${CLICKHOUSE_CLIENT} --format=JSONEachRow -nq " system flush logs; +-- Show queries sorted by start time. +select attribute['db.statement'] as query, + attribute['clickhouse.query_status'] as status, + attribute['clickhouse.tracestate'] as tracestate, + 1 as sorted_by_start_time + from system.opentelemetry_span_log + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + order by start_time_us + ; + +-- Show queries sorted by finish time. +select attribute['db.statement'] as query, + attribute['clickhouse.query_status'] as query_status, + attribute['clickhouse.tracestate'] as tracestate, + 1 as sorted_by_finish_time + from system.opentelemetry_span_log + where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + and operation_name = 'query' + order by finish_time_us + ; + -- Check the number of query spans with given trace id, to verify it was -- propagated. select count(*) "'"'"total spans"'"'", @@ -89,10 +111,10 @@ check_log echo "===sampled===" query_id=$(${CLICKHOUSE_CLIENT} -q "select lower(hex(reverse(reinterpretAsString(generateUUIDv4()))))") -for i in {1..200} +for i in {1..20} do ${CLICKHOUSE_CLIENT} \ - --opentelemetry_start_trace_probability=0.1 \ + --opentelemetry_start_trace_probability=0.5 \ --query_id "$query_id-$i" \ --query "select 1 from remote('127.0.0.2', system, one) format Null" \ & @@ -108,8 +130,8 @@ wait ${CLICKHOUSE_CLIENT} -q "system flush logs" ${CLICKHOUSE_CLIENT} -q " - -- expect 200 * 0.1 = 20 sampled events on average - select if(count() > 1 and count() < 50, 'OK', 'Fail') + -- expect 20 * 0.5 = 10 sampled events on average + select if(2 <= count() and count() <= 18, 'OK', 'Fail') from system.opentelemetry_span_log where operation_name = 'query' and parent_span_id = 0 -- only account for the initial queries From 941cbc43f358562bbf7067d0e49050a6d0410399 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 18 Aug 2021 11:28:22 +0000 Subject: [PATCH 183/220] Fix --- src/Common/filesystemHelpers.cpp | 19 +++++++++++++++++++ src/Common/filesystemHelpers.h | 2 ++ src/Dictionaries/LibraryDictionarySource.cpp | 9 ++++++--- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 4855500b776..730099f4476 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -27,6 +27,7 @@ namespace ErrorCodes extern const int CANNOT_STATVFS; extern const int PATH_ACCESS_DENIED; extern const int CANNOT_CREATE_FILE; + extern const int BAD_ARGUMENTS; } @@ -122,6 +123,17 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p return path_starts_with_prefix_path; } +bool symlinkStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path) +{ + auto absolute_path = std::filesystem::absolute(path); + auto absolute_prefix_path = std::filesystem::weakly_canonical(prefix_path); + + auto [_, prefix_path_mismatch_it] = std::mismatch(absolute_path.begin(), absolute_path.end(), absolute_prefix_path.begin(), absolute_prefix_path.end()); + + bool path_starts_with_prefix_path = (prefix_path_mismatch_it == absolute_prefix_path.end()); + return path_starts_with_prefix_path; +} + bool pathStartsWith(const String & path, const String & prefix_path) { auto filesystem_path = std::filesystem::path(path); @@ -130,6 +142,13 @@ bool pathStartsWith(const String & path, const String & prefix_path) return pathStartsWith(filesystem_path, filesystem_prefix_path); } +bool symlinkStartsWith(const String & path, const String & prefix_path) +{ + auto filesystem_path = std::filesystem::path(path); + auto filesystem_prefix_path = std::filesystem::path(prefix_path); + + return symlinkStartsWith(filesystem_path, filesystem_prefix_path); +} } diff --git a/src/Common/filesystemHelpers.h b/src/Common/filesystemHelpers.h index b7525a64fae..71ef7844ef7 100644 --- a/src/Common/filesystemHelpers.h +++ b/src/Common/filesystemHelpers.h @@ -35,6 +35,8 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p /// Returns true if path starts with prefix path bool pathStartsWith(const String & path, const String & prefix_path); +bool symlinkStartsWith(const String & path, const String & prefix_path); + } namespace FS diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 288abde8788..f2c5cefa543 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -41,10 +41,13 @@ LibraryDictionarySource::LibraryDictionarySource( , sample_block{sample_block_} , context(Context::createCopy(context_)) { - if (fs::path(path).is_relative()) - path = fs::canonical(path); + bool path_checked = false; + if (fs::is_symlink(path)) + path_checked = symlinkStartsWith(path, context->getDictionariesLibPath()); + else + path_checked = pathStartsWith(path, context->getDictionariesLibPath()); - if (created_from_ddl && !pathStartsWith(path, context->getDictionariesLibPath())) + if (created_from_ddl && !path_checked) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, context->getDictionariesLibPath()); if (!fs::exists(path)) From 8149653d1716e0f64373304a0556427a5d11250e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 18 Aug 2021 14:45:53 +0300 Subject: [PATCH 184/220] Update version_date.tsv after release 21.7.8.58 --- utils/list-versions/version_date.tsv | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 1bc21bfff17..799eaaf1c05 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,5 +1,6 @@ v21.8.4.51-lts 2021-08-17 v21.8.3.44-lts 2021-08-12 +v21.7.8.58-stable 2021-08-17 v21.7.7.47-stable 2021-08-09 v21.7.6.39-stable 2021-08-06 v21.7.5.29-stable 2021-07-28 From 31d75c9c385d3e5755118ce6d93834b4c9283b81 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 18 Aug 2021 15:15:31 +0300 Subject: [PATCH 185/220] fix split build --- base/daemon/SentryWriter.cpp | 2 +- programs/keeper/Keeper.cpp | 2 +- programs/server/Server.cpp | 2 +- src/{Common => Core}/ServerUUID.cpp | 2 +- src/{Common => Core}/ServerUUID.h | 0 src/Functions/serverUUID.cpp | 2 +- 6 files changed, 5 insertions(+), 5 deletions(-) rename src/{Common => Core}/ServerUUID.cpp (98%) rename src/{Common => Core}/ServerUUID.h (100%) diff --git a/base/daemon/SentryWriter.cpp b/base/daemon/SentryWriter.cpp index de772afdec3..470e1deb362 100644 --- a/base/daemon/SentryWriter.cpp +++ b/base/daemon/SentryWriter.cpp @@ -12,7 +12,7 @@ #include #include #include -#include +#include #if !defined(ARCADIA_BUILD) # include "Common/config_version.h" diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index c35e3e64d37..4d01a523853 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -17,7 +17,7 @@ #include #include #include -#include +#include #include #include diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 7e2c250d6e5..b7f0be4b85a 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -39,7 +39,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Common/ServerUUID.cpp b/src/Core/ServerUUID.cpp similarity index 98% rename from src/Common/ServerUUID.cpp rename to src/Core/ServerUUID.cpp index 486b0206e56..721c406ff5f 100644 --- a/src/Common/ServerUUID.cpp +++ b/src/Core/ServerUUID.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include diff --git a/src/Common/ServerUUID.h b/src/Core/ServerUUID.h similarity index 100% rename from src/Common/ServerUUID.h rename to src/Core/ServerUUID.h diff --git a/src/Functions/serverUUID.cpp b/src/Functions/serverUUID.cpp index 988142aed5d..4b70b1576ac 100644 --- a/src/Functions/serverUUID.cpp +++ b/src/Functions/serverUUID.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include From bcfab277e7a26d418b5ec5bb6798ef9b3cad4910 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Wed, 18 Aug 2021 15:25:54 +0300 Subject: [PATCH 186/220] Update ReadBufferFromKafkaConsumer.cpp --- src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp index f5f1974dcfe..86037276166 100644 --- a/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp +++ b/src/Storages/Kafka/ReadBufferFromKafkaConsumer.cpp @@ -472,12 +472,11 @@ bool ReadBufferFromKafkaConsumer::nextImpl() allowed = false; ++current; - // in some cases message can be NULL (tombstone records for example) - // parsers are not ready to get NULLs on input. - if (unlikely(message_data == nullptr)) + /// If message is empty, return end of stream. + if (message_data == nullptr) return false; - // XXX: very fishy place with const casting. + /// const_cast is needed, because ReadBuffer works with non-const char *. auto * new_position = reinterpret_cast(const_cast(message_data)); BufferBase::set(new_position, message_size, 0); return true; From 0ed046eb7b36988962244fe06fc423ef9175d1c3 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Wed, 18 Aug 2021 15:33:11 +0300 Subject: [PATCH 187/220] remove irrelevant comments --- src/Storages/StorageReplicatedMergeTree.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index bdec69095ce..98ce2ac73e1 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -4325,7 +4325,6 @@ void StorageReplicatedMergeTree::startup() restarting_thread.start(); /// Wait while restarting_thread initializes LeaderElection (and so on) or makes first attempt to do it - /// TODO Do we still need startup_event? startup_event.wait(); startBackgroundMovesIfNeeded(); @@ -7206,7 +7205,6 @@ MutationCommands StorageReplicatedMergeTree::getFirstAlterMutationCommandsForPar void StorageReplicatedMergeTree::startBackgroundMovesIfNeeded() { - /// FIXME is it related to replication somehow? If it is we should start it from RestartingThread only if (areBackgroundMovesNeeded()) background_moves_executor.start(); } From 2433ae65b5cd019f7c2022412893743b5faa4dab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 18 Aug 2021 15:48:24 +0200 Subject: [PATCH 188/220] Mention from_env in the documentation --- docs/en/operations/configuration-files.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/en/operations/configuration-files.md b/docs/en/operations/configuration-files.md index 5c942efc77f..44f9353063c 100644 --- a/docs/en/operations/configuration-files.md +++ b/docs/en/operations/configuration-files.md @@ -18,6 +18,18 @@ Some settings specified in the main configuration file can be overridden in othe - If `replace` is specified, it replaces the entire element with the specified one. - If `remove` is specified, it deletes the element. +You can also declare attributes as coming from environment variables by using `from_env="VARIABLE_NAME"`: + +```xml + + + + + + + +``` + ## Substitution {#substitution} The config can also define “substitutions”. If an element has the `incl` attribute, the corresponding substitution from the file will be used as the value. By default, the path to the file with substitutions is `/etc/metrika.xml`. This can be changed in the [include_from](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-include_from) element in the server config. The substitution values are specified in `/yandex/substitution_name` elements in this file. If a substitution specified in `incl` does not exist, it is recorded in the log. To prevent ClickHouse from logging missing substitutions, specify the `optional="true"` attribute (for example, settings for [macros](../operations/server-configuration-parameters/settings.md)). From c71cac2af37e558792629126606c6c1dfec7075e Mon Sep 17 00:00:00 2001 From: Kruglov Pavel <48961922+Avogar@users.noreply.github.com> Date: Wed, 18 Aug 2021 16:55:13 +0300 Subject: [PATCH 189/220] Fix test --- tests/queries/0_stateless/01822_short_circuit.reference | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/01822_short_circuit.reference b/tests/queries/0_stateless/01822_short_circuit.reference index d78605adbf0..ec805b2aa76 100644 --- a/tests/queries/0_stateless/01822_short_circuit.reference +++ b/tests/queries/0_stateless/01822_short_circuit.reference @@ -1394,12 +1394,12 @@ Decimal32 42 21 14 -10 +10.5 0 42 21 14 -10 +10.5 +\N \N \N -\N \ No newline at end of file From 94d68ee8ac5fdb24e7b805142c90c345d3b39280 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 18 Aug 2021 14:14:53 +0000 Subject: [PATCH 190/220] Fix, add test --- src/Common/filesystemHelpers.cpp | 1 + tests/integration/test_library_bridge/test.py | 44 +++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 730099f4476..86ae7a046be 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -126,6 +126,7 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p bool symlinkStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path) { auto absolute_path = std::filesystem::absolute(path); + absolute_path = absolute_path.lexically_normal(); auto absolute_prefix_path = std::filesystem::weakly_canonical(prefix_path); auto [_, prefix_path_mismatch_it] = std::mismatch(absolute_path.begin(), absolute_path.end(), absolute_prefix_path.begin(), absolute_prefix_path.end()); diff --git a/tests/integration/test_library_bridge/test.py b/tests/integration/test_library_bridge/test.py index 97b2ccfbdbe..12a967ebaa4 100644 --- a/tests/integration/test_library_bridge/test.py +++ b/tests/integration/test_library_bridge/test.py @@ -44,6 +44,11 @@ def ch_cluster(): '/usr/bin/g++ -shared -o /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp'], user='root') + instance.exec_in_container( + ['bash', '-c', + '/usr/bin/g++ -shared -o /dict_lib_copy.so -fPIC /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib.cpp'], user='root') + instance.exec_in_container(['bash', '-c', 'ln -s /dict_lib_copy.so /etc/clickhouse-server/config.d/dictionaries_lib/dict_lib_symlink.so']) + yield cluster finally: @@ -59,6 +64,7 @@ def test_load_all(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + instance.query('DROP DICTIONARY IF EXISTS lib_dict') instance.query(''' CREATE DICTIONARY lib_dict (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key @@ -128,6 +134,7 @@ def test_load_keys(ch_cluster): if instance.is_built_with_memory_sanitizer(): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + instance.query('DROP DICTIONARY IF EXISTS lib_dict_ckc') instance.query(''' CREATE DICTIONARY lib_dict_ckc (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) PRIMARY KEY key @@ -148,6 +155,7 @@ def test_load_all_many_rows(ch_cluster): pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") num_rows = [1000, 10000, 100000, 1000000] + instance.query('DROP DICTIONARY IF EXISTS lib_dict') for num in num_rows: instance.query(''' CREATE DICTIONARY lib_dict (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) @@ -267,6 +275,42 @@ def test_bridge_dies_with_parent(ch_cluster): instance.query('DROP DICTIONARY lib_dict_c') +def test_path_validation(ch_cluster): + if instance.is_built_with_memory_sanitizer(): + pytest.skip("Memory Sanitizer cannot work with third-party shared libraries") + + instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') + instance.query(''' + CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) + PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/dict_lib_symlink.so')) + LAYOUT(CACHE( + SIZE_IN_CELLS 10000000 + BLOCK_SIZE 4096 + FILE_SIZE 16777216 + READ_BUFFER_SIZE 1048576 + MAX_STORED_KEYS 1048576)) + LIFETIME(2) ; + ''') + + result = instance.query('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert(result.strip() == '101') + + instance.query('DROP DICTIONARY IF EXISTS lib_dict_c') + instance.query(''' + CREATE DICTIONARY lib_dict_c (key UInt64, value1 UInt64, value2 UInt64, value3 UInt64) + PRIMARY KEY key SOURCE(library(PATH '/etc/clickhouse-server/config.d/dictionaries_lib/../../../../dict_lib_copy.so')) + LAYOUT(CACHE( + SIZE_IN_CELLS 10000000 + BLOCK_SIZE 4096 + FILE_SIZE 16777216 + READ_BUFFER_SIZE 1048576 + MAX_STORED_KEYS 1048576)) + LIFETIME(2) ; + ''') + result = instance.query_and_get_error('''select dictGet(lib_dict_c, 'value1', toUInt64(1));''') + assert('DB::Exception: File path /etc/clickhouse-server/config.d/dictionaries_lib/../../../../dict_lib_copy.so is not inside /etc/clickhouse-server/config.d/dictionaries_lib' in result) + + if __name__ == '__main__': cluster.start() input("Cluster created, press any key to destroy...") From df3ba20642b67c59782fe66cf2ba0efc5fbc9ada Mon Sep 17 00:00:00 2001 From: Denny Crane Date: Wed, 18 Aug 2021 12:05:43 -0300 Subject: [PATCH 191/220] Update InterpreterCreateQuery.cpp Fix setting name "allow_experimental_database_materialized_postgresql" --- src/Interpreters/InterpreterCreateQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 4c1a3064c3d..76cb6c783ba 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -223,7 +223,7 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) if (create.storage->engine->name == "MaterializedPostgreSQL" && !getContext()->getSettingsRef().allow_experimental_database_materialized_postgresql && !internal) { throw Exception("MaterializedPostgreSQL is an experimental database engine. " - "Enable allow_experimental_database_postgresql_replica to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); + "Enable allow_experimental_database_materialized_postgresql to use it.", ErrorCodes::UNKNOWN_DATABASE_ENGINE); } DatabasePtr database = DatabaseFactory::get(create, metadata_path / "", getContext()); From 08fb4ede355bfdccb15cebaee54ae7ea1d5b98d5 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Wed, 18 Aug 2021 19:05:27 +0300 Subject: [PATCH 192/220] Update Server.cpp --- programs/server/Server.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index b7f0be4b85a..c3623eca007 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -146,8 +146,6 @@ static bool jemallocOptionEnabled(const char *name) static bool jemallocOptionEnabled(const char *) { return 0; } #endif -namespace fs = std::filesystem; - int mainEntryClickHouseServer(int argc, char ** argv) { DB::Server app; From 240398d58c70be52631390214dcc19e139847a5d Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Wed, 18 Aug 2021 19:13:07 +0300 Subject: [PATCH 193/220] Fix bug in expand() method --- src/Columns/ColumnArray.cpp | 2 +- src/Columns/ColumnFixedString.cpp | 2 +- src/Columns/ColumnString.cpp | 2 +- src/Columns/MaskOperations.cpp | 2 +- .../0_stateless/01822_short_circuit.reference | 880 +++++++++++++----- .../0_stateless/01822_short_circuit.sql | 20 + 6 files changed, 664 insertions(+), 244 deletions(-) diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 1601fb1ff94..7c7c6a4d9db 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -565,7 +565,7 @@ void ColumnArray::expand(const IColumn::Filter & mask, bool inverted) while (index >= 0) { offsets_data[index] = last_offset; - if (mask[index] ^ inverted) + if (!!mask[index] ^ inverted) { if (from < 0) throw Exception("Too many bytes in mask", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index e818e974493..ce4f11a38d6 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -354,7 +354,7 @@ void ColumnFixedString::expand(const IColumn::Filter & mask, bool inverted) chars.resize_fill(mask.size() * n, 0); while (index >= 0) { - if (mask[index] ^ inverted) + if (!!mask[index] ^ inverted) { if (from < 0) throw Exception("Too many bytes in mask", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index 2f5903abfc1..d02fa66baf4 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -178,7 +178,7 @@ void ColumnString::expand(const IColumn::Filter & mask, bool inverted) while (index >= 0) { offsets_data[index] = last_offset; - if (mask[index] ^ inverted) + if (!!mask[index] ^ inverted) { if (from < 0) throw Exception("Too many bytes in mask", ErrorCodes::LOGICAL_ERROR); diff --git a/src/Columns/MaskOperations.cpp b/src/Columns/MaskOperations.cpp index 6852c895d51..759d0af7127 100644 --- a/src/Columns/MaskOperations.cpp +++ b/src/Columns/MaskOperations.cpp @@ -26,7 +26,7 @@ void expandDataByMask(PaddedPODArray & data, const PaddedPODArray & ma data.resize(mask.size()); while (index >= 0) { - if (mask[index] ^ inverted) + if (!!mask[index] ^ inverted) { if (from < 0) throw Exception("Too many bytes in mask", ErrorCodes::LOGICAL_ERROR); diff --git a/tests/queries/0_stateless/01822_short_circuit.reference b/tests/queries/0_stateless/01822_short_circuit.reference index ec805b2aa76..96c4e161244 100644 --- a/tests/queries/0_stateless/01822_short_circuit.reference +++ b/tests/queries/0_stateless/01822_short_circuit.reference @@ -100,26 +100,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -139,26 +119,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -220,26 +180,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -259,26 +199,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -340,26 +260,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -379,26 +279,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -460,26 +340,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -499,26 +359,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -580,26 +420,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -619,26 +439,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -700,6 +500,26 @@ 18 19 0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 2 3 4 @@ -759,6 +579,426 @@ 17 18 19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N 00 22 33 @@ -819,6 +1059,26 @@ 1717 1818 1919 +00 +\N +\N +\N +\N +55 +\N +\N +\N +\N +1010 +\N +\N +\N +\N +1515 +\N +\N +\N +\N 10 12 13 @@ -879,6 +1139,26 @@ 27 28 29 +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +20 +\N +\N +\N +\N +25 +\N +\N +\N +\N 1970-01-01 1970-01-01 1970-01-01 @@ -939,6 +1219,26 @@ 1970-01-01 1970-01-01 1970-01-01 +1970-01-01 +\N +\N +\N +\N +1970-01-01 +\N +\N +\N +\N +1970-01-01 +\N +\N +\N +\N +1970-01-01 +\N +\N +\N +\N 1970-01-01 00:00:00 1970-01-01 05:33:20 1970-01-01 08:20:00 @@ -999,6 +1299,26 @@ 1970-01-02 23:13:20 1970-01-03 02:00:00 1970-01-03 04:46:40 +1970-01-01 00:00:00 +\N +\N +\N +\N +1970-01-01 13:53:20 +\N +\N +\N +\N +1970-01-02 03:46:40 +\N +\N +\N +\N +1970-01-02 17:40:00 +\N +\N +\N +\N 0 2 3 @@ -1060,26 +1380,6 @@ 18 19 0 -2 -3 -4 -5 -5 -7 -8 -9 -10 -10 -12 -13 -14 -15 -15 -17 -18 -19 -20 -0 \N \N \N @@ -1099,26 +1399,6 @@ \N \N \N -\N -1 -2 -3 -4 -\N -6 -7 -8 -9 -\N -11 -12 -13 -14 -\N -16 -17 -18 -19 0 2 3 @@ -1180,6 +1460,26 @@ 18 19 0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 2 3 4 @@ -1239,6 +1539,106 @@ 17 18 19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +0 +2 +3 +4 +5 +5 +7 +8 +9 +10 +10 +12 +13 +14 +15 +15 +17 +18 +19 +20 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N +\N +1 +2 +3 +4 +\N +6 +7 +8 +9 +\N +11 +12 +13 +14 +\N +16 +17 +18 +19 +0 +\N +\N +\N +\N +5 +\N +\N +\N +\N +10 +\N +\N +\N +\N +15 +\N +\N +\N +\N [] [0,1] [0,1,2] diff --git a/tests/queries/0_stateless/01822_short_circuit.sql b/tests/queries/0_stateless/01822_short_circuit.sql index db50721a468..fe8a0315d4a 100644 --- a/tests/queries/0_stateless/01822_short_circuit.sql +++ b/tests/queries/0_stateless/01822_short_circuit.sql @@ -11,82 +11,102 @@ select count() from (select if(number >= 0, number, sleep(1)) from numbers(10000 select if(number % 5 == 0, toInt8OrZero(toString(number)), toInt8OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt8OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt8OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt8OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt8OrZero(toString(number)), toUInt8OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt8OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt8OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt8OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toInt32OrZero(toString(number)), toInt32OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt32OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt32OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt32OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt32OrZero(toString(number)), toUInt32OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt32OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt32OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt32OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toInt64OrZero(toString(number)), toInt64OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt64OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt64OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt64OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt64OrZero(toString(number)), toUInt64OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt64OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt64OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt64OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toInt128OrZero(toString(number)), toInt128OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt128OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt128OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt128OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt128OrZero(toString(number)), toUInt128OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt128OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt128OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt128OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toInt256OrZero(toString(number)), toInt256OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toInt256OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toInt256OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toInt256OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toUInt256OrZero(toString(number)), toUInt256OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toUInt256OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toUInt256OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toUInt256OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toFloat32OrZero(toString(number)), toFloat32OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toFloat32OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toFloat32OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toFloat32OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toFloat64OrZero(toString(number)), toFloat64OrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toFloat64OrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toFloat64OrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toFloat64OrZero(toString(number))) from numbers(20); select if(number % 5 == 0, repeat(toString(number), 2), repeat(toString(number + 1), 2)) from numbers(20); select if(number % 5 == 0, repeat(toString(number), 2), Null) from numbers(20); select if(number % 5 == 0, Null, repeat(toString(number), 2)) from numbers(20); +select if(number % 5, Null, repeat(toString(number), 2)) from numbers(20); select if(number % 5 == 0, toFixedString(toString(number + 10), 2), toFixedString(toString(number + 11), 2)) from numbers(20); select if(number % 5 == 0, toFixedString(toString(number + 10), 2), Null) from numbers(20); select if(number % 5 == 0, Null, toFixedString(toString(number + 10), 2)) from numbers(20); +select if(number % 5, Null, toFixedString(toString(number + 10), 2)) from numbers(20); select if(number % 5 == 0, toDateOrZero(toString(number)), toDateOrZero(toString(number + 1))) from numbers(20); select if(number % 5 == 0, toDateOrZero(toString(number)), Null) from numbers(20); select if(number % 5 == 0, Null, toDateOrZero(toString(number))) from numbers(20); +select if(number % 5, Null, toDateOrZero(toString(number))) from numbers(20); select if(number % 5 == 0, toDateTimeOrZero(toString(number * 10000), 'UTC'), toDateTimeOrZero(toString((number + 1) * 10000), 'UTC')) from numbers(20); select if(number % 5 == 0, toDateTimeOrZero(toString(number * 10000), 'UTC'), Null) from numbers(20); select if(number % 5 == 0, Null, toDateTimeOrZero(toString(number * 10000), 'UTC')) from numbers(20); +select if(number % 5, Null, toDateTimeOrZero(toString(number * 10000), 'UTC')) from numbers(20); select if(number % 5 == 0, toDecimal32OrZero(toString(number), 5), toDecimal32OrZero(toString(number + 1), 5)) from numbers(20); select if(number % 5 == 0, toDecimal32OrZero(toString(number), 5), Null) from numbers(20); select if(number % 5 == 0, Null, toDecimal32OrZero(toString(number), 5)) from numbers(20); +select if(number % 5, Null, toDecimal32OrZero(toString(number), 5)) from numbers(20); select if(number % 5 == 0, toDecimal64OrZero(toString(number), 5), toDecimal64OrZero(toString(number + 1), 5)) from numbers(20); select if(number % 5 == 0, toDecimal64OrZero(toString(number), 5), Null) from numbers(20); select if(number % 5 == 0, Null, toDecimal64OrZero(toString(number), 5)) from numbers(20); +select if(number % 5, Null, toDecimal64OrZero(toString(number), 5)) from numbers(20); select if(number % 5 == 0, toDecimal128OrZero(toString(number), 5), toDecimal128OrZero(toString(number + 1), 5)) from numbers(20); select if(number % 5 == 0, toDecimal128OrZero(toString(number), 5), Null) from numbers(20); select if(number % 5 == 0, Null, toDecimal128OrZero(toString(number), 5)) from numbers(20); +select if(number % 5, Null, toDecimal128OrZero(toString(number), 5)) from numbers(20); select if(number % 5 == 0, toDecimal256OrZero(toString(number), 5), toDecimal256OrZero(toString(number + 1), 5)) from numbers(20); select if(number % 5 == 0, toDecimal256OrZero(toString(number), 5), Null) from numbers(20); select if(number % 5 == 0, Null, toDecimal256OrZero(toString(number), 5)) from numbers(20); +select if(number % 5, Null, toDecimal256OrZero(toString(number), 5)) from numbers(20); select if(number % 5 == 0, range(number), range(number + 1)) from numbers(20); select if(number % 5 == 0, replicate(toString(number), range(number)), replicate(toString(number), range(number + 1))) from numbers(20); From baa7b204fc66cac28793eac2e8e0995ec66c1ad5 Mon Sep 17 00:00:00 2001 From: Dmitriy Date: Wed, 18 Aug 2021 19:27:39 +0300 Subject: [PATCH 194/220] Fix by comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Поправил согласно комментариям в PR. --- .../operations/system-tables/zookeeper_log.md | 27 +++++++++---------- .../operations/system-tables/zookeeper_log.md | 23 +++++++--------- 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/docs/en/operations/system-tables/zookeeper_log.md b/docs/en/operations/system-tables/zookeeper_log.md index 5585b1a6dcd..f7d86c6689b 100644 --- a/docs/en/operations/system-tables/zookeeper_log.md +++ b/docs/en/operations/system-tables/zookeeper_log.md @@ -1,25 +1,22 @@ # system.zookeeper_log {#system-zookeeper_log} -This table contains information about the parameters of the request to the ZooKeeper client and the response from it. +This table contains information about the parameters of the request to the ZooKeeper server and the response from it. For requests, only columns with request parameters are filled in, and the remaining columns are filled with default values (`0` or `NULL`). When the response arrives, the data from the response is added to the other columns. -!!! info "Note" - The table does not exist if ZooKeeper is not configured. - Columns with request parameters: - `type` ([Enum](../../sql-reference/data-types/enum.md)) — Event type in the ZooKeeper client. Can have one of the following values: - - `request` — The request has been sent. - - `response` — The response was received. - - `finalize` — The connection is lost, no response was received. -- `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the request was completed. -- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the request was completed. -- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the request. -- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The client port that was used to make the request. + - `Request` — The request has been sent. + - `Response` — The response was received. + - `Finalize` — The connection is lost, no response was received. +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — The date when the event happened. +- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — The date and time when the event happened. +- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address of ZooKeeper server that was used to make the request. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — The port of ZooKeeper server that was used to make the request. - `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — The session ID that the ZooKeeper server sets for each connection. -- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request line and the paired `response`/`finalize` line. -- `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The request whether the [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches) has been installed. +- `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — The ID of the request within the session. This is usually a sequential request number. It is the same for the request row and the paired `response`/`finalize` row. +- `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — The request whether the [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches) has been set. - `op_num` ([Enum](../../sql-reference/data-types/enum.md)) — The type of request or response. - `path` ([String](../../sql-reference/data-types/string.md)) — The path to the ZooKeeper node specified in the request, or an empty string if the request not requires specifying a path. - `data` ([String](../../sql-reference/data-types/string.md)) — The data written to the ZooKeeper node (for the `SET` and `CREATE` requests — what the request wanted to write, for the response to the `GET` request — what was read) or an empty string. @@ -32,8 +29,8 @@ Columns with request parameters: Columns with request response parameters: - `zxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — ZooKeeper transaction ID. The serial number issued by the ZooKeeper server in response to a successfully executed request (`0` if the request was not executed/returned an error/the client does not know whether the request was executed). -- `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — Error code. Can have one of the following values: - - `ZOK` — The response to the request was received. +- `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — Error code. Can have many values, here are just some of them: + - `ZOK` — The request was executed seccessfully. - `ZCONNECTIONLOSS` — The connection was lost. - `ZOPERATIONTIMEOUT` — The request execution timeout has expired. - `ZSESSIONEXPIRED` — The session has expired. diff --git a/docs/ru/operations/system-tables/zookeeper_log.md b/docs/ru/operations/system-tables/zookeeper_log.md index a78a5089bdf..16f02cb0489 100644 --- a/docs/ru/operations/system-tables/zookeeper_log.md +++ b/docs/ru/operations/system-tables/zookeeper_log.md @@ -1,22 +1,19 @@ # system.zookeeper_log {#system-zookeeper_log} -Эта таблица содержит информацию о параметрах запроса к клиенту ZooKeeper и ответа от него. +Эта таблица содержит информацию о параметрах запроса к серверу ZooKeeper и ответа от него. Для запросов заполняются только столбцы с параметрами запроса, а остальные столбцы заполняются значениями по умолчанию (`0` или `NULL`). Когда поступает ответ, данные добавляются в столбцы с параметрами ответа на запрос. -!!! info "Примечание" - Таблицы не существует, если ZooKeeper не сконфигурирован. - Столбцы с параметрами запроса: - `type` ([Enum](../../sql-reference/data-types/enum.md)) — тип события в клиенте ZooKeeper. Может иметь одно из следующих значений: - - `request` — запрос отправлен. - - `response` — ответ получен. - - `finalize` — соединение разорвано, ответ не получен. -- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата завершения выполнения запроса. -- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — дата и время завершения выполнения запроса. -- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес, с которого был сделан запрос. -- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт, с которого был сделан запрос. + - `Request` — запрос отправлен. + - `Response` — ответ получен. + - `Finalize` — соединение разорвано, ответ не получен. +- `event_date` ([Date](../../sql-reference/data-types/date.md)) — дата, когда произошло событие. +- `event_time` ([DateTime64](../../sql-reference/data-types/datetime64.md)) — дата и время, когда произошло событие. +- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP адрес сервера ZooKeeper, с которого был сделан запрос. +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md)) — порт сервера ZooKeeper, с которого был сделан запрос. - `session_id` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор сессии, который сервер ZooKeeper создает для каждого соединения. - `xid` ([Int32](../../sql-reference/data-types/int-uint.md)) — идентификатор запроса внутри сессии. Обычно это последовательный номер запроса, одинаковый у строки запроса и у парной строки `response`/`finalize`. - `has_watch` ([UInt8](../../sql-reference/data-types/int-uint.md)) — установлен ли запрос [watch](https://zookeeper.apache.org/doc/r3.3.3/zookeeperProgrammers.html#ch_zkWatches). @@ -32,8 +29,8 @@ Столбцы с параметрами ответа на запрос: - `zxid` ([Int64](../../sql-reference/data-types/int-uint.md)) — идентификатор транзакции в ZooKeeper. Последовательный номер, выданный сервером ZooKeeper в ответе на успешно выполненный запрос (`0` — запрос не был выполнен, возвращена ошибка или клиент ZooKeeper не знает, был ли выполнен запрос). -- `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — код ошибки. Может иметь одно из следующих значений: - - `ZOK` — получен ответ на запрос. +- `error` ([Nullable(Enum)](../../sql-reference/data-types/nullable.md)) — код ошибки. Может иметь много значений, здесь приведены только некоторые из них: + - `ZOK` — запрос успешно выполнен. - `ZCONNECTIONLOSS` — соединение разорвано. - `ZOPERATIONTIMEOUT` — истекло время ожидания выполнения запроса. - `ZSESSIONEXPIRED` — истекло время сессии. From 4d4dae79f0ecb122b3fb9304ffb85eee7edc0c4d Mon Sep 17 00:00:00 2001 From: ubuntu Date: Thu, 19 Aug 2021 00:47:40 +0800 Subject: [PATCH 195/220] fix: build issue --- .../functions/type-conversion-functions.md | 8 ++-- src/Functions/FunctionSnowflake.h | 44 +++++++++---------- src/Functions/dateTime64ToSnowflake.cpp | 14 ------ src/Functions/dateTimeToSnowflake.cpp | 14 ------ src/Functions/snowflake.cpp | 34 ++++++++++++++ src/Functions/snowflakeToDateTime.cpp | 14 ------ src/Functions/snowflakeToDateTime64.cpp | 14 ------ 7 files changed, 58 insertions(+), 84 deletions(-) delete mode 100644 src/Functions/dateTime64ToSnowflake.cpp delete mode 100644 src/Functions/dateTimeToSnowflake.cpp create mode 100644 src/Functions/snowflake.cpp delete mode 100644 src/Functions/snowflakeToDateTime.cpp delete mode 100644 src/Functions/snowflakeToDateTime64.cpp diff --git a/docs/en/sql-reference/functions/type-conversion-functions.md b/docs/en/sql-reference/functions/type-conversion-functions.md index 4f1a2d49d23..ad6edaea312 100644 --- a/docs/en/sql-reference/functions/type-conversion-functions.md +++ b/docs/en/sql-reference/functions/type-conversion-functions.md @@ -1342,7 +1342,7 @@ Result: ## snowflakeToDateTime {#snowflakeToDateTime} -extract time from snowflake id as DateTime format. +Extract time from snowflake id as DateTime format. **Syntax** @@ -1378,7 +1378,7 @@ Result: ## snowflakeToDateTime64 {#snowflakeToDateTime64} -extract time from snowflake id as DateTime64 format. +Extract time from snowflake id as DateTime64 format. **Syntax** @@ -1414,7 +1414,7 @@ Result: ## dateTimeToSnowflake {#dateTimeToSnowflake} -convert DateTime to the first snowflake id at the giving time. +Convert DateTime to the first snowflake id at the giving time. **Syntax** @@ -1452,7 +1452,7 @@ Result: ## dateTime64ToSnowflake {#dateTime64ToSnowflake} -convert DateTime64 to the first snowflake id at the giving time. +Convert DateTime64 to the first snowflake id at the giving time. **Syntax** diff --git a/src/Functions/FunctionSnowflake.h b/src/Functions/FunctionSnowflake.h index 3dd378e4956..3f0f404f7e4 100644 --- a/src/Functions/FunctionSnowflake.h +++ b/src/Functions/FunctionSnowflake.h @@ -12,23 +12,28 @@ namespace DB { - namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; } +/** According to Twitter's post on Snowflake, we can extract the timestamp for a snowflake ID by right shifting + * the snowflake ID by 22 bits(10 bits machine ID and 12 bits sequence ID) and adding the Twitter epoch time of 1288834974657. + * https://en.wikipedia.org/wiki/Snowflake_ID + * https://blog.twitter.com/engineering/en_us/a/2010/announcing-snowflake + * https://ws-dl.blogspot.com/2019/08/2019-08-03-tweetedat-finding-tweet.html +*/ +static constexpr long snowflake_epoch = 1288834974657L; +static constexpr int time_shift = 22; class FunctionDateTimeToSnowflake : public IFunction { private: const char * name; + public: - FunctionDateTimeToSnowflake(const char * name_) - : name(name_) - { - } + FunctionDateTimeToSnowflake(const char * name_) : name(name_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } @@ -54,7 +59,7 @@ public: const auto & source_data = typeid_cast(col).getData(); for (size_t i = 0; i < input_rows_count; ++i) { - result_data[i] = (int64_t(source_data[i])*1000-1288834974657)<<22; + result_data[i] = (Int64(source_data[i]) * 1000 - snowflake_epoch) << time_shift; } return res_column; @@ -66,11 +71,9 @@ class FunctionSnowflakeToDateTime : public IFunction { private: const char * name; + public: - FunctionSnowflakeToDateTime(const char * name_) - : name(name_) - { - } + FunctionSnowflakeToDateTime(const char * name_) : name(name_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -104,9 +107,8 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { - result_data[i] = ((source_data[i]>>22)+1288834974657)/1000; + result_data[i] = ((source_data[i] >> time_shift) + snowflake_epoch) / 1000; } - return res_column; } }; @@ -116,11 +118,9 @@ class FunctionDateTime64ToSnowflake : public IFunction { private: const char * name; + public: - FunctionDateTime64ToSnowflake(const char * name_) - : name(name_) - { - } + FunctionDateTime64ToSnowflake(const char * name_) : name(name_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 1; } @@ -146,7 +146,7 @@ public: const auto & source_data = typeid_cast &>(col).getData(); for (size_t i = 0; i < input_rows_count; ++i) { - result_data[i] = (source_data[i]-1288834974657)<<22; + result_data[i] = (source_data[i] - snowflake_epoch) << time_shift; } return res_column; @@ -158,11 +158,9 @@ class FunctionSnowflakeToDateTime64 : public IFunction { private: const char * name; + public: - FunctionSnowflakeToDateTime64(const char * name_) - : name(name_) - { - } + FunctionSnowflakeToDateTime64(const char * name_) : name(name_) { } String getName() const override { return name; } size_t getNumberOfArguments() const override { return 0; } @@ -171,7 +169,6 @@ public: DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { - if (arguments.size() < 1 || arguments.size() > 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); @@ -197,9 +194,8 @@ public: for (size_t i = 0; i < input_rows_count; ++i) { - result_data[i] = (source_data[i]>>22)+1288834974657; + result_data[i] = (source_data[i] >> time_shift) + snowflake_epoch; } - return res_column; } }; diff --git a/src/Functions/dateTime64ToSnowflake.cpp b/src/Functions/dateTime64ToSnowflake.cpp deleted file mode 100644 index 87e35c25371..00000000000 --- a/src/Functions/dateTime64ToSnowflake.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include - -namespace DB -{ - -void registerDateTime64ToSnowflake(FunctionFactory & factory) -{ - factory.registerFunction("dateTime64ToSnowflake", - [](ContextPtr){ return std::make_unique( - std::make_shared("dateTime64ToSnowflake")); }); -} - -} diff --git a/src/Functions/dateTimeToSnowflake.cpp b/src/Functions/dateTimeToSnowflake.cpp deleted file mode 100644 index 246f35cc1dc..00000000000 --- a/src/Functions/dateTimeToSnowflake.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include - -namespace DB -{ - -void registerDateTimeToSnowflake(FunctionFactory & factory) -{ - factory.registerFunction("dateTimeToSnowflake", - [](ContextPtr){ return std::make_unique( - std::make_shared("dateTimeToSnowflake")); }); -} - -} diff --git a/src/Functions/snowflake.cpp b/src/Functions/snowflake.cpp new file mode 100644 index 00000000000..5ac1d229d17 --- /dev/null +++ b/src/Functions/snowflake.cpp @@ -0,0 +1,34 @@ +#include +#include + +namespace DB +{ + +void registerDateTimeToSnowflake(FunctionFactory & factory) +{ + factory.registerFunction("dateTimeToSnowflake", + [](ContextPtr){ return std::make_unique( + std::make_shared("dateTimeToSnowflake")); }); +} + +void registerDateTime64ToSnowflake(FunctionFactory & factory) +{ + factory.registerFunction("dateTime64ToSnowflake", + [](ContextPtr){ return std::make_unique( + std::make_shared("dateTime64ToSnowflake")); }); +} + +void registerSnowflakeToDateTime(FunctionFactory & factory) +{ + factory.registerFunction("snowflakeToDateTime", + [](ContextPtr){ return std::make_unique( + std::make_shared("snowflakeToDateTime")); }); +} +void registerSnowflakeToDateTime64(FunctionFactory & factory) +{ + factory.registerFunction("snowflakeToDateTime64", + [](ContextPtr){ return std::make_unique( + std::make_shared("snowflakeToDateTime64")); }); +} + +} diff --git a/src/Functions/snowflakeToDateTime.cpp b/src/Functions/snowflakeToDateTime.cpp deleted file mode 100644 index 37f5e07512f..00000000000 --- a/src/Functions/snowflakeToDateTime.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include - -namespace DB -{ - -void registerSnowflakeToDateTime(FunctionFactory & factory) -{ - factory.registerFunction("snowflakeToDateTime", - [](ContextPtr){ return std::make_unique( - std::make_shared("snowflakeToDateTime")); }); -} - -} diff --git a/src/Functions/snowflakeToDateTime64.cpp b/src/Functions/snowflakeToDateTime64.cpp deleted file mode 100644 index ef9502a224e..00000000000 --- a/src/Functions/snowflakeToDateTime64.cpp +++ /dev/null @@ -1,14 +0,0 @@ -#include -#include - -namespace DB -{ - -void registerSnowflakeToDateTime64(FunctionFactory & factory) -{ - factory.registerFunction("snowflakeToDateTime64", - [](ContextPtr){ return std::make_unique( - std::make_shared("snowflakeToDateTime64")); }); -} - -} From f5a91e5e5f63b6ab3c2e77f950b96b227ad6b318 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Wed, 18 Aug 2021 23:17:00 +0300 Subject: [PATCH 196/220] Fix style check --- src/Common/filesystemHelpers.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 86ae7a046be..5bed3ea1120 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -27,7 +27,6 @@ namespace ErrorCodes extern const int CANNOT_STATVFS; extern const int PATH_ACCESS_DENIED; extern const int CANNOT_CREATE_FILE; - extern const int BAD_ARGUMENTS; } From 7f15c5c55e48b5d050eeb1177733a48e22e4fc20 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 19 Aug 2021 01:21:23 +0300 Subject: [PATCH 197/220] Update entrypoint.sh --- docker/test/performance-comparison/entrypoint.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docker/test/performance-comparison/entrypoint.sh b/docker/test/performance-comparison/entrypoint.sh index 19af56e3299..1295e5567fb 100755 --- a/docker/test/performance-comparison/entrypoint.sh +++ b/docker/test/performance-comparison/entrypoint.sh @@ -127,8 +127,14 @@ export PATH export REF_PR export REF_SHA +# Try to collect some core dumps. I've seen two patterns in Sandbox: +# 1) |/home/zomb-sandbox/venv/bin/python /home/zomb-sandbox/client/sandbox/bin/coredumper.py %e %p %g %u %s %P %c +# Not sure what this script does (puts them to sandbox resources, logs some messages?), +# and it's not accessible from inside docker anyway. +# 2) something like %e.%p.core.dmp. The dump should end up in the workspace directory. +# At least we remove the ulimit and then try to pack some common file names into output. ulimit -c unlimited - +cat /proc/sys/kernel/core_pattern # Start the main comparison script. { \ @@ -147,13 +153,11 @@ done dmesg -T > dmesg.log -cat /proc/sys/kernel/core_pattern - ls -lath 7z a '-x!*/tmp' /output/output.7z ./*.{log,tsv,html,txt,rep,svg,columns} \ {right,left}/{performance,scripts} {{right,left}/db,db0}/preprocessed_configs \ report analyze benchmark metrics \ - ./*.core.dmp + ./*.core.dmp ./*.core cp compare.log /output From db53638a95532f31d99d9664cfab619e8510a2ce Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov <36882414+akuzm@users.noreply.github.com> Date: Thu, 19 Aug 2021 01:21:51 +0300 Subject: [PATCH 198/220] Update download.sh --- docker/test/performance-comparison/download.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/download.sh b/docker/test/performance-comparison/download.sh index bd72547ec1c..49323c28700 100755 --- a/docker/test/performance-comparison/download.sh +++ b/docker/test/performance-comparison/download.sh @@ -13,7 +13,7 @@ left_sha=$2 # right_pr=$3 not used for now right_sha=$4 -datasets=${CHPC_DATASETS:-"hits1 hits10 hits100 values"} +datasets=${CHPC_DATASETS-"hits1 hits10 hits100 values"} declare -A dataset_paths dataset_paths["hits10"]="https://s3.mds.yandex.net/clickhouse-private-datasets/hits_10m_single/partitions/hits_10m_single.tar" From 3db3b40b5e3af3b9f31635ca4f7f0c181d1fda4c Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 19 Aug 2021 01:38:12 +0300 Subject: [PATCH 199/220] Revert "less sys calls #2: make vdso work again" --- base/glibc-compatibility/musl/getauxval.c | 49 ++++++----------------- 1 file changed, 12 insertions(+), 37 deletions(-) diff --git a/base/glibc-compatibility/musl/getauxval.c b/base/glibc-compatibility/musl/getauxval.c index dad7aa938d7..a429273fa1a 100644 --- a/base/glibc-compatibility/musl/getauxval.c +++ b/base/glibc-compatibility/musl/getauxval.c @@ -1,5 +1,4 @@ #include -#include "atomic.h" #include // __environ #include @@ -18,7 +17,18 @@ static size_t __find_auxv(unsigned long type) return (size_t) -1; } -unsigned long __getauxval(unsigned long type) +__attribute__((constructor)) static void __auxv_init() +{ + size_t i; + for (i = 0; __environ[i]; i++); + __auxv = (unsigned long *) (__environ + i + 1); + + size_t secure_idx = __find_auxv(AT_SECURE); + if (secure_idx != ((size_t) -1)) + __auxv_secure = __auxv[secure_idx]; +} + +unsigned long getauxval(unsigned long type) { if (type == AT_SECURE) return __auxv_secure; @@ -33,38 +43,3 @@ unsigned long __getauxval(unsigned long type) errno = ENOENT; return 0; } - -static void * volatile getauxval_func; - -static unsigned long __auxv_init(unsigned long type) -{ - if (!__environ) - { - // __environ is not initialized yet so we can't initialize __auxv right now. - // That's normally occurred only when getauxval() is called from some sanitizer's internal code. - errno = ENOENT; - return 0; - } - - // Initialize __auxv and __auxv_secure. - size_t i; - for (i = 0; __environ[i]; i++); - __auxv = (unsigned long *) (__environ + i + 1); - - size_t secure_idx = __find_auxv(AT_SECURE); - if (secure_idx != ((size_t) -1)) - __auxv_secure = __auxv[secure_idx]; - - // Now we've initialized __auxv, next time getauxval() will only call __get_auxval(). - a_cas_p(&getauxval_func, (void *)__auxv_init, (void *)__getauxval); - - return __getauxval(type); -} - -// First time getauxval() will call __auxv_init(). -static void * volatile getauxval_func = (void *)__auxv_init; - -unsigned long getauxval(unsigned long type) -{ - return ((unsigned long (*)(unsigned long))getauxval_func)(type); -} From 3b09640d941d7d0bd1133821204e715a56fe1b67 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Thu, 12 Aug 2021 18:16:55 +0300 Subject: [PATCH 200/220] Use sessions more. --- programs/local/LocalServer.cpp | 14 +++--- .../CassandraDictionarySource.cpp | 2 +- .../ClickHouseDictionarySource.cpp | 31 +++++++----- src/Dictionaries/ClickHouseDictionarySource.h | 4 +- src/Dictionaries/DictionaryFactory.cpp | 15 +++--- src/Dictionaries/DictionaryFactory.h | 6 +-- src/Dictionaries/DictionarySourceFactory.cpp | 4 +- src/Dictionaries/DictionarySourceFactory.h | 4 +- src/Dictionaries/DictionarySourceHelpers.cpp | 48 +++++++++++-------- src/Dictionaries/DictionarySourceHelpers.h | 7 ++- src/Dictionaries/DirectDictionary.cpp | 2 +- .../ExecutableDictionarySource.cpp | 8 ++-- .../ExecutablePoolDictionarySource.cpp | 12 ++--- src/Dictionaries/FileDictionarySource.cpp | 6 +-- src/Dictionaries/FlatDictionary.cpp | 2 +- src/Dictionaries/HTTPDictionarySource.cpp | 6 +-- src/Dictionaries/HashedDictionary.cpp | 8 ++-- src/Dictionaries/IPAddressDictionary.cpp | 2 +- src/Dictionaries/LibraryDictionarySource.cpp | 4 +- src/Dictionaries/MySQLDictionarySource.cpp | 4 +- .../PolygonDictionaryImplementations.cpp | 2 +- .../PostgreSQLDictionarySource.cpp | 8 ++-- src/Dictionaries/RangeHashedDictionary.cpp | 2 +- src/Dictionaries/RedisDictionarySource.cpp | 2 +- src/Dictionaries/XDBCDictionarySource.cpp | 10 ++-- .../registerCacheDictionaries.cpp | 22 ++++----- src/Interpreters/Context.cpp | 21 -------- src/Interpreters/Context.h | 8 +--- .../ExternalDictionariesLoader.cpp | 8 ++++ 29 files changed, 131 insertions(+), 141 deletions(-) diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 44e9880fabb..957bda4d75c 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -374,14 +375,13 @@ void LocalServer::processQueries() if (!parse_res.second) throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR); - /// we can't mutate global global_context (can lead to races, as it was already passed to some background threads) - /// so we can't reuse it safely as a query context and need a copy here - auto context = Context::createCopy(global_context); + /// Authenticate and create a context to execute queries. + Session session{global_context, ClientInfo::Interface::TCP}; + session.authenticate("default", "", Poco::Net::SocketAddress{}); - context->makeSessionContext(); - context->makeQueryContext(); - - context->authenticate("default", "", Poco::Net::SocketAddress{}); + /// Use the same context for all queries. + auto context = session.makeQueryContext(); + context->makeSessionContext(); /// initial_create_query requires a session context to be set. context->setCurrentQueryId(""); applyCmdSettings(context); diff --git a/src/Dictionaries/CassandraDictionarySource.cpp b/src/Dictionaries/CassandraDictionarySource.cpp index aa8d6107508..d9a4dd0fd22 100644 --- a/src/Dictionaries/CassandraDictionarySource.cpp +++ b/src/Dictionaries/CassandraDictionarySource.cpp @@ -17,7 +17,7 @@ void registerDictionarySourceCassandra(DictionarySourceFactory & factory) [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, [[maybe_unused]] const std::string & config_prefix, [[maybe_unused]] Block & sample_block, - ContextPtr /* context */, + ContextPtr /* global_context */, const std::string & /* default_database */, bool /*created_from_ddl*/) -> DictionarySourcePtr { diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index d4f01dee8b2..b09a7b14cc4 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -63,19 +64,18 @@ ClickHouseDictionarySource::ClickHouseDictionarySource( const DictionaryStructure & dict_struct_, const Configuration & configuration_, const Block & sample_block_, - ContextPtr context_) + ContextMutablePtr context_, + std::shared_ptr local_session_) : update_time{std::chrono::system_clock::from_time_t(0)} , dict_struct{dict_struct_} , configuration{configuration_} , query_builder{dict_struct, configuration.db, "", configuration.table, configuration.query, configuration.where, IdentifierQuotingStyle::Backticks} , sample_block{sample_block_} - , context(Context::createCopy(context_)) + , local_session(local_session_) + , context(context_) , pool{createPool(configuration)} , load_all_query{query_builder.composeLoadAllQuery()} { - /// Query context is needed because some code in executeQuery function may assume it exists. - /// Current example is Context::getSampleBlockCache from InterpreterSelectWithUnionQuery::getSampleBlock. - context->makeQueryContext(); } ClickHouseDictionarySource::ClickHouseDictionarySource(const ClickHouseDictionarySource & other) @@ -85,11 +85,11 @@ ClickHouseDictionarySource::ClickHouseDictionarySource(const ClickHouseDictionar , invalidate_query_response{other.invalidate_query_response} , query_builder{dict_struct, configuration.db, "", configuration.table, configuration.query, configuration.where, IdentifierQuotingStyle::Backticks} , sample_block{other.sample_block} + , local_session(other.local_session) , context(Context::createCopy(other.context)) , pool{createPool(configuration)} , load_all_query{other.load_all_query} { - context->makeQueryContext(); } std::string ClickHouseDictionarySource::getUpdateFieldAndDate() @@ -222,14 +222,13 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & default_database [[maybe_unused]], bool /* created_from_ddl */) -> DictionarySourcePtr { bool secure = config.getBool(config_prefix + ".secure", false); - auto context_copy = Context::createCopy(context); - UInt16 default_port = getPortFromContext(context_copy, secure); + UInt16 default_port = getPortFromContext(global_context, secure); std::string settings_config_prefix = config_prefix + ".clickhouse"; std::string host = config.getString(settings_config_prefix + ".host", "localhost"); @@ -252,12 +251,18 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) .secure = config.getBool(settings_config_prefix + ".secure", false) }; - /// We should set user info even for the case when the dictionary is loaded in-process (without TCP communication). + ContextMutablePtr context; + std::shared_ptr local_session; if (configuration.is_local) { - context_copy->authenticate(configuration.user, configuration.password, Poco::Net::SocketAddress("127.0.0.1", 0)); - context_copy = copyContextAndApplySettings(config_prefix, context_copy, config); + /// Start local session in case when the dictionary is loaded in-process (without TCP communication). + local_session = std::make_shared(global_context, ClientInfo::Interface::TCP); + local_session->authenticate(configuration.user, configuration.password, Poco::Net::SocketAddress{"127.0.0.1", 0}); + context = local_session->makeQueryContext(); + context->applySettingsChanges(readSettingsFromDictionaryConfig(config, config_prefix)); } + else + context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); String dictionary_name = config.getString(".dictionary.name", ""); String dictionary_database = config.getString(".dictionary.database", ""); @@ -265,7 +270,7 @@ void registerDictionarySourceClickHouse(DictionarySourceFactory & factory) if (dictionary_name == configuration.table && dictionary_database == configuration.db) throw Exception(ErrorCodes::BAD_ARGUMENTS, "ClickHouseDictionarySource table cannot be dictionary table"); - return std::make_unique(dict_struct, configuration, sample_block, context_copy); + return std::make_unique(dict_struct, configuration, sample_block, context, local_session); }; factory.registerSource("clickhouse", create_table_source); diff --git a/src/Dictionaries/ClickHouseDictionarySource.h b/src/Dictionaries/ClickHouseDictionarySource.h index 2daa296af3e..58243e43b15 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.h +++ b/src/Dictionaries/ClickHouseDictionarySource.h @@ -39,7 +39,8 @@ public: const DictionaryStructure & dict_struct_, const Configuration & configuration_, const Block & sample_block_, - ContextPtr context); + ContextMutablePtr context_, + std::shared_ptr local_session_); /// copy-constructor is provided in order to support cloneability ClickHouseDictionarySource(const ClickHouseDictionarySource & other); @@ -81,6 +82,7 @@ private: mutable std::string invalidate_query_response; ExternalQueryBuilder query_builder; Block sample_block; + std::shared_ptr local_session; ContextMutablePtr context; ConnectionPoolWithFailoverPtr pool; const std::string load_all_query; diff --git a/src/Dictionaries/DictionaryFactory.cpp b/src/Dictionaries/DictionaryFactory.cpp index 62b28ed7d14..4cab42c9445 100644 --- a/src/Dictionaries/DictionaryFactory.cpp +++ b/src/Dictionaries/DictionaryFactory.cpp @@ -31,7 +31,7 @@ DictionaryPtr DictionaryFactory::create( const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) const { Poco::Util::AbstractConfiguration::Keys keys; @@ -45,12 +45,9 @@ DictionaryPtr DictionaryFactory::create( const DictionaryStructure dict_struct{config, config_prefix}; DictionarySourcePtr source_ptr = DictionarySourceFactory::instance().create( - name, config, config_prefix + ".source", dict_struct, context, config.getString(config_prefix + ".database", ""), created_from_ddl); + name, config, config_prefix + ".source", dict_struct, global_context, config.getString(config_prefix + ".database", ""), created_from_ddl); LOG_TRACE(&Poco::Logger::get("DictionaryFactory"), "Created dictionary source '{}' for dictionary '{}'", source_ptr->toString(), name); - if (context->hasQueryContext() && context->getSettingsRef().log_queries) - context->getQueryContext()->addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, name); - const auto & layout_type = keys.front(); { @@ -58,7 +55,7 @@ DictionaryPtr DictionaryFactory::create( if (found != registered_layouts.end()) { const auto & layout_creator = found->second.layout_create_function; - return layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr), context, created_from_ddl); + return layout_creator(name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); } } @@ -68,10 +65,10 @@ DictionaryPtr DictionaryFactory::create( layout_type); } -DictionaryPtr DictionaryFactory::create(const std::string & name, const ASTCreateQuery & ast, ContextPtr context) const +DictionaryPtr DictionaryFactory::create(const std::string & name, const ASTCreateQuery & ast, ContextPtr global_context) const { - auto configuration = getDictionaryConfigurationFromAST(ast, context); - return DictionaryFactory::create(name, *configuration, "dictionary", context, true); + auto configuration = getDictionaryConfigurationFromAST(ast, global_context); + return DictionaryFactory::create(name, *configuration, "dictionary", global_context, true); } bool DictionaryFactory::isComplex(const std::string & layout_type) const diff --git a/src/Dictionaries/DictionaryFactory.h b/src/Dictionaries/DictionaryFactory.h index b717009aa8a..b1dad340f4b 100644 --- a/src/Dictionaries/DictionaryFactory.h +++ b/src/Dictionaries/DictionaryFactory.h @@ -36,13 +36,13 @@ public: const std::string & name, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) const; /// Create dictionary from DDL-query DictionaryPtr create(const std::string & name, const ASTCreateQuery & ast, - ContextPtr context) const; + ContextPtr global_context) const; using LayoutCreateFunction = std::function; bool isComplex(const std::string & layout_type) const; diff --git a/src/Dictionaries/DictionarySourceFactory.cpp b/src/Dictionaries/DictionarySourceFactory.cpp index 1992c87d31f..80b60f29e37 100644 --- a/src/Dictionaries/DictionarySourceFactory.cpp +++ b/src/Dictionaries/DictionarySourceFactory.cpp @@ -80,7 +80,7 @@ DictionarySourcePtr DictionarySourceFactory::create( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const DictionaryStructure & dict_struct, - ContextPtr context, + ContextPtr global_context, const std::string & default_database, bool check_config) const { @@ -99,7 +99,7 @@ DictionarySourcePtr DictionarySourceFactory::create( { const auto & create_source = found->second; auto sample_block = createSampleBlock(dict_struct); - return create_source(dict_struct, config, config_prefix, sample_block, context, default_database, check_config); + return create_source(dict_struct, config, config_prefix, sample_block, global_context, default_database, check_config); } throw Exception(ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG, diff --git a/src/Dictionaries/DictionarySourceFactory.h b/src/Dictionaries/DictionarySourceFactory.h index bb583927ac4..f4c3fa12163 100644 --- a/src/Dictionaries/DictionarySourceFactory.h +++ b/src/Dictionaries/DictionarySourceFactory.h @@ -35,7 +35,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & default_database, bool check_config)>; @@ -48,7 +48,7 @@ public: const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, const DictionaryStructure & dict_struct, - ContextPtr context, + ContextPtr global_context, const std::string & default_database, bool check_config) const; diff --git a/src/Dictionaries/DictionarySourceHelpers.cpp b/src/Dictionaries/DictionarySourceHelpers.cpp index 092e7187e8f..cf003dceb8e 100644 --- a/src/Dictionaries/DictionarySourceHelpers.cpp +++ b/src/Dictionaries/DictionarySourceHelpers.cpp @@ -59,30 +59,36 @@ Block blockForKeys( return block; } -ContextMutablePtr copyContextAndApplySettings( - const std::string & config_prefix, - ContextPtr context, - const Poco::Util::AbstractConfiguration & config) + +SettingsChanges readSettingsFromDictionaryConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) { - auto local_context = Context::createCopy(context); - if (config.has(config_prefix + ".settings")) + if (!config.has(config_prefix + ".settings")) + return {}; + + const auto prefix = config_prefix + ".settings"; + + Poco::Util::AbstractConfiguration::Keys config_keys; + config.keys(prefix, config_keys); + + SettingsChanges changes; + + for (const std::string & key : config_keys) { - const auto prefix = config_prefix + ".settings"; - - Poco::Util::AbstractConfiguration::Keys config_keys; - config.keys(prefix, config_keys); - - SettingsChanges changes; - - for (const std::string & key : config_keys) - { - const auto value = config.getString(prefix + "." + key); - changes.emplace_back(key, value); - } - - local_context->applySettingsChanges(changes); + const auto value = config.getString(prefix + "." + key); + changes.emplace_back(key, value); } - return local_context; + + return changes; +} + + +ContextMutablePtr copyContextAndApplySettingsFromDictionaryConfig( + const ContextPtr & context, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix) +{ + auto context_copy = Context::createCopy(context); + auto changes = readSettingsFromDictionaryConfig(config, config_prefix); + context_copy->applySettingsChanges(changes); + return context_copy; } static Block transformHeader(Block header, Block block_to_add) diff --git a/src/Dictionaries/DictionarySourceHelpers.h b/src/Dictionaries/DictionarySourceHelpers.h index b955b6ffb66..5470321745a 100644 --- a/src/Dictionaries/DictionarySourceHelpers.h +++ b/src/Dictionaries/DictionarySourceHelpers.h @@ -14,6 +14,7 @@ namespace DB { struct DictionaryStructure; +class SettingsChanges; /// For simple key @@ -29,10 +30,8 @@ Block blockForKeys( const std::vector & requested_rows); /// Used for applying settings to copied context in some register[...]Source functions -ContextMutablePtr copyContextAndApplySettings( - const std::string & config_prefix, - ContextPtr context, - const Poco::Util::AbstractConfiguration & config); +SettingsChanges readSettingsFromDictionaryConfig(const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); +ContextMutablePtr copyContextAndApplySettingsFromDictionaryConfig(const ContextPtr & context, const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix); /** A stream, adds additional columns to each block that it will read from inner stream. * diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index e12100a556d..10e7414b42f 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -307,7 +307,7 @@ namespace const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /* created_from_ddl */) { const auto * layout_name = dictionary_key_type == DictionaryKeyType::Simple ? "direct" : "complex_key_direct"; diff --git a/src/Dictionaries/ExecutableDictionarySource.cpp b/src/Dictionaries/ExecutableDictionarySource.cpp index 8802d04ff30..5c6add34f1f 100644 --- a/src/Dictionaries/ExecutableDictionarySource.cpp +++ b/src/Dictionaries/ExecutableDictionarySource.cpp @@ -275,7 +275,7 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { @@ -285,10 +285,10 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) /// Executable dictionaries may execute arbitrary commands. /// It's OK for dictionaries created by administrator from xml-file, but /// maybe dangerous for dictionaries created from DDL-queries. - if (created_from_ddl && context->getApplicationType() != Context::ApplicationType::LOCAL) + if (created_from_ddl && global_context->getApplicationType() != Context::ApplicationType::LOCAL) throw Exception(ErrorCodes::DICTIONARY_ACCESS_DENIED, "Dictionaries with executable dictionary source are not allowed to be created from DDL query"); - auto context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); std::string settings_config_prefix = config_prefix + ".executable"; @@ -301,7 +301,7 @@ void registerDictionarySourceExecutable(DictionarySourceFactory & factory) .implicit_key = config.getBool(settings_config_prefix + ".implicit_key", false) }; - return std::make_unique(dict_struct, configuration, sample_block, context_local_copy); + return std::make_unique(dict_struct, configuration, sample_block, context); }; factory.registerSource("executable", create_table_source); diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.cpp b/src/Dictionaries/ExecutablePoolDictionarySource.cpp index e97b4253407..e18664e477c 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp +++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp @@ -279,7 +279,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { @@ -289,17 +289,15 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) /// Executable dictionaries may execute arbitrary commands. /// It's OK for dictionaries created by administrator from xml-file, but /// maybe dangerous for dictionaries created from DDL-queries. - if (created_from_ddl && context->getApplicationType() != Context::ApplicationType::LOCAL) + if (created_from_ddl && global_context->getApplicationType() != Context::ApplicationType::LOCAL) throw Exception(ErrorCodes::DICTIONARY_ACCESS_DENIED, "Dictionaries with executable pool dictionary source are not allowed to be created from DDL query"); - auto context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + ContextMutablePtr context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); /** Currently parallel parsing input format cannot read exactly max_block_size rows from input, * so it will be blocked on ReadBufferFromFileDescriptor because this file descriptor represent pipe that does not have eof. */ - auto settings_no_parallel_parsing = context_local_copy->getSettings(); - settings_no_parallel_parsing.input_format_parallel_parsing = false; - context_local_copy->setSettings(settings_no_parallel_parsing); + context->setSetting("input_format_parallel_parsing", Field{false}); String settings_config_prefix = config_prefix + ".executable_pool"; @@ -319,7 +317,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) .implicit_key = config.getBool(settings_config_prefix + ".implicit_key", false), }; - return std::make_unique(dict_struct, configuration, sample_block, context_local_copy); + return std::make_unique(dict_struct, configuration, sample_block, context); }; factory.registerSource("executable_pool", create_table_source); diff --git a/src/Dictionaries/FileDictionarySource.cpp b/src/Dictionaries/FileDictionarySource.cpp index bea14d88d1e..54ce5e4a448 100644 --- a/src/Dictionaries/FileDictionarySource.cpp +++ b/src/Dictionaries/FileDictionarySource.cpp @@ -77,7 +77,7 @@ void registerDictionarySourceFile(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { @@ -87,9 +87,9 @@ void registerDictionarySourceFile(DictionarySourceFactory & factory) const auto filepath = config.getString(config_prefix + ".file.path"); const auto format = config.getString(config_prefix + ".file.format"); - auto context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + const auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); - return std::make_unique(filepath, format, sample_block, context_local_copy, created_from_ddl); + return std::make_unique(filepath, format, sample_block, context, created_from_ddl); }; factory.registerSource("file", create_table_source); diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index 26667db1081..5ecf3299ea6 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -557,7 +557,7 @@ void registerDictionaryFlat(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /* created_from_ddl */) -> DictionaryPtr { if (dict_struct.key) diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index 4a80ebdf975..b5cf59b4474 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -213,13 +213,13 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { if (dict_struct.has_expressions) throw Exception(ErrorCodes::LOGICAL_ERROR, "Dictionary source of type `http` does not support attribute expressions"); - auto context_local_copy = copyContextAndApplySettings(config_prefix, context, config); + auto context = copyContextAndApplySettingsFromDictionaryConfig(global_context, config, config_prefix); const auto & settings_config_prefix = config_prefix + ".http"; const auto & credentials_prefix = settings_config_prefix + ".credentials"; @@ -258,7 +258,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) .header_entries = std::move(header_entries) }; - return std::make_unique(dict_struct, configuration, credentials, sample_block, context_local_copy, created_from_ddl); + return std::make_unique(dict_struct, configuration, credentials, sample_block, context, created_from_ddl); }; factory.registerSource("http", create_table_source); } diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index d462631fba8..fd5865e24c0 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -756,13 +756,13 @@ void registerDictionaryHashed(DictionaryFactory & factory) using namespace std::placeholders; factory.registerLayout("hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ false); }, false); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* global_context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ false); }, false); factory.registerLayout("sparse_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ true); }, false); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* global_context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Simple, /* sparse = */ true); }, false); factory.registerLayout("complex_key_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ false); }, true); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* global_context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ false); }, true); factory.registerLayout("complex_key_sparse_hashed", - [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ true); }, true); + [=](auto && a, auto && b, auto && c, auto && d, DictionarySourcePtr e, ContextPtr /* global_context */, bool /*created_from_ddl*/){ return create_layout(a, b, c, d, std::move(e), DictionaryKeyType::Complex, /* sparse = */ true); }, true); } diff --git a/src/Dictionaries/IPAddressDictionary.cpp b/src/Dictionaries/IPAddressDictionary.cpp index fbe911c1d49..b688362d048 100644 --- a/src/Dictionaries/IPAddressDictionary.cpp +++ b/src/Dictionaries/IPAddressDictionary.cpp @@ -954,7 +954,7 @@ void registerDictionaryTrie(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /*created_from_ddl*/) -> DictionaryPtr { if (!dict_struct.key || dict_struct.key->size() != 1) diff --git a/src/Dictionaries/LibraryDictionarySource.cpp b/src/Dictionaries/LibraryDictionarySource.cpp index 288abde8788..602e0c5b08e 100644 --- a/src/Dictionaries/LibraryDictionarySource.cpp +++ b/src/Dictionaries/LibraryDictionarySource.cpp @@ -183,11 +183,11 @@ void registerDictionarySourceLibrary(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool created_from_ddl) -> DictionarySourcePtr { - return std::make_unique(dict_struct, config, config_prefix + ".library", sample_block, context, created_from_ddl); + return std::make_unique(dict_struct, config, config_prefix + ".library", sample_block, global_context, created_from_ddl); }; factory.registerSource("library", create_table_source); diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index 2eebb6970d0..bd53c1e60a7 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -31,11 +31,11 @@ void registerDictionarySourceMysql(DictionarySourceFactory & factory) [[maybe_unused]] const Poco::Util::AbstractConfiguration & config, [[maybe_unused]] const std::string & config_prefix, [[maybe_unused]] Block & sample_block, - [[maybe_unused]] ContextPtr context, + [[maybe_unused]] ContextPtr global_context, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { #if USE_MYSQL - StreamSettings mysql_input_stream_settings(context->getSettingsRef() + StreamSettings mysql_input_stream_settings(global_context->getSettingsRef() , config.getBool(config_prefix + ".mysql.close_connection", false) || config.getBool(config_prefix + ".mysql.share_connection", false) , false , config.getBool(config_prefix + ".mysql.fail_on_connection_loss", false) ? 1 : default_num_tries_on_connection_loss); diff --git a/src/Dictionaries/PolygonDictionaryImplementations.cpp b/src/Dictionaries/PolygonDictionaryImplementations.cpp index 7c3eb421a4a..72869ad57ba 100644 --- a/src/Dictionaries/PolygonDictionaryImplementations.cpp +++ b/src/Dictionaries/PolygonDictionaryImplementations.cpp @@ -167,7 +167,7 @@ DictionaryPtr createLayout(const std::string & , const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /*created_from_ddl*/) { const String database = config.getString(config_prefix + ".database", ""); diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index 5a546820959..3fe9e899cd9 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -182,7 +182,7 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { @@ -190,8 +190,8 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) const auto settings_config_prefix = config_prefix + ".postgresql"; auto pool = std::make_shared( config, settings_config_prefix, - context->getSettingsRef().postgresql_connection_pool_size, - context->getSettingsRef().postgresql_connection_pool_wait_timeout); + global_context->getSettingsRef().postgresql_connection_pool_size, + global_context->getSettingsRef().postgresql_connection_pool_wait_timeout); PostgreSQLDictionarySource::Configuration configuration { @@ -211,7 +211,7 @@ void registerDictionarySourcePostgreSQL(DictionarySourceFactory & factory) (void)config; (void)config_prefix; (void)sample_block; - (void)context; + (void)global_context; throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Dictionary source of type `postgresql` is disabled because ClickHouse was built without postgresql support."); #endif diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index ea0af493bdf..390871661c7 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -688,7 +688,7 @@ void registerDictionaryRangeHashed(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr /* context */, + ContextPtr /* global_context */, bool /*created_from_ddl*/) -> DictionaryPtr { if (dict_struct.key) diff --git a/src/Dictionaries/RedisDictionarySource.cpp b/src/Dictionaries/RedisDictionarySource.cpp index 6561a122e9d..24a14d8cc80 100644 --- a/src/Dictionaries/RedisDictionarySource.cpp +++ b/src/Dictionaries/RedisDictionarySource.cpp @@ -12,7 +12,7 @@ void registerDictionarySourceRedis(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const String & config_prefix, Block & sample_block, - ContextPtr /* context */, + ContextPtr /* global_context */, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { return std::make_unique(dict_struct, config, config_prefix + ".redis", sample_block); diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index e79e55910b7..9fc7e92634b 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -234,12 +234,12 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, Block & sample_block, - ContextPtr context, + ContextPtr global_context, const std::string & /* default_database */, bool /* check_config */) -> DictionarySourcePtr { #if USE_ODBC BridgeHelperPtr bridge = std::make_shared>( - context, context->getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string")); + global_context, global_context->getSettings().http_receive_timeout, config.getString(config_prefix + ".odbc.connection_string")); std::string settings_config_prefix = config_prefix + ".odbc"; @@ -255,13 +255,13 @@ void registerDictionarySourceXDBC(DictionarySourceFactory & factory) .update_lag = config.getUInt64(settings_config_prefix + ".update_lag", 1) }; - return std::make_unique(dict_struct, configuration, sample_block, context, bridge); + return std::make_unique(dict_struct, configuration, sample_block, global_context, bridge); #else (void)dict_struct; (void)config; (void)config_prefix; (void)sample_block; - (void)context; + (void)global_context; throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Dictionary source of type `odbc` is disabled because poco library was built without ODBC support."); #endif @@ -276,7 +276,7 @@ void registerDictionarySourceJDBC(DictionarySourceFactory & factory) const Poco::Util::AbstractConfiguration & /* config */, const std::string & /* config_prefix */, Block & /* sample_block */, - ContextPtr /* context */, + ContextPtr /* global_context */, const std::string & /* default_database */, bool /* created_from_ddl */) -> DictionarySourcePtr { throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, diff --git a/src/Dictionaries/registerCacheDictionaries.cpp b/src/Dictionaries/registerCacheDictionaries.cpp index 64c1c55e0ba..69197f992f0 100644 --- a/src/Dictionaries/registerCacheDictionaries.cpp +++ b/src/Dictionaries/registerCacheDictionaries.cpp @@ -154,7 +154,7 @@ DictionaryPtr createCacheDictionaryLayout( const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context [[maybe_unused]], + ContextPtr global_context [[maybe_unused]], bool created_from_ddl [[maybe_unused]]) { String layout_type; @@ -213,8 +213,8 @@ DictionaryPtr createCacheDictionaryLayout( else { auto storage_configuration = parseSSDCacheStorageConfiguration(config, full_name, layout_type, dictionary_layout_prefix, dict_lifetime); - if (created_from_ddl && !pathStartsWith(storage_configuration.file_path, context->getUserFilesPath())) - throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", storage_configuration.file_path, context->getUserFilesPath()); + if (created_from_ddl && !pathStartsWith(storage_configuration.file_path, global_context->getUserFilesPath())) + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", storage_configuration.file_path, global_context->getUserFilesPath()); storage = std::make_shared>(storage_configuration); } @@ -239,10 +239,10 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); }; factory.registerLayout("cache", create_simple_cache_layout, false); @@ -252,10 +252,10 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); }; factory.registerLayout("complex_key_cache", create_complex_key_cache_layout, true); @@ -267,10 +267,10 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); }; factory.registerLayout("ssd_cache", create_simple_ssd_cache_layout, false); @@ -280,9 +280,9 @@ void registerDictionaryCache(DictionaryFactory & factory) const Poco::Util::AbstractConfiguration & config, const std::string & config_prefix, DictionarySourcePtr source_ptr, - ContextPtr context, + ContextPtr global_context, bool created_from_ddl) -> DictionaryPtr { - return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), std::move(context), created_from_ddl); + return createCacheDictionaryLayout(full_name, dict_struct, config, config_prefix, std::move(source_ptr), global_context, created_from_ddl); }; factory.registerLayout("complex_key_ssd_cache", create_complex_key_ssd_cache_layout, true); diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index a634c19dcd6..84a858d8603 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -589,27 +589,6 @@ ConfigurationPtr Context::getUsersConfig() } -void Context::authenticate(const String & name, const String & password, const Poco::Net::SocketAddress & address) -{ - authenticate(BasicCredentials(name, password), address); -} - -void Context::authenticate(const Credentials & credentials, const Poco::Net::SocketAddress & address) -{ - auto authenticated_user_id = getAccessControlManager().login(credentials, address.host()); - - client_info.current_user = credentials.getUserName(); - client_info.current_address = address; - -#if defined(ARCADIA_BUILD) - /// This is harmful field that is used only in foreign "Arcadia" build. - if (const auto * basic_credentials = dynamic_cast(&credentials)) - client_info.current_password = basic_credentials->getPassword(); -#endif - - setUser(authenticated_user_id); -} - void Context::setUser(const UUID & user_id_) { auto lock = getLock(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 4e378dacf01..1b636deb532 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -362,13 +362,9 @@ public: void setUsersConfig(const ConfigurationPtr & config); ConfigurationPtr getUsersConfig(); - /// Sets the current user, checks the credentials and that the specified address is allowed to connect from. - /// The function throws an exception if there is no such user or password is wrong. - void authenticate(const String & user_name, const String & password, const Poco::Net::SocketAddress & address); - void authenticate(const Credentials & credentials, const Poco::Net::SocketAddress & address); - /// Sets the current user assuming that he/she is already authenticated. - /// WARNING: This function doesn't check password! Don't use until it's necessary! + /// WARNING: This function doesn't check password! + /// Normally you shouldn't call this function. Use the Session class to do authentication instead. void setUser(const UUID & user_id_); UserPtr getUser() const; diff --git a/src/Interpreters/ExternalDictionariesLoader.cpp b/src/Interpreters/ExternalDictionariesLoader.cpp index 83931649443..cbb0e52b91b 100644 --- a/src/Interpreters/ExternalDictionariesLoader.cpp +++ b/src/Interpreters/ExternalDictionariesLoader.cpp @@ -45,12 +45,20 @@ ExternalLoader::LoadablePtr ExternalDictionariesLoader::create( ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::getDictionary(const std::string & dictionary_name, ContextPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); + + if (local_context->hasQueryContext() && local_context->getSettingsRef().log_queries) + local_context->addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, resolved_dictionary_name); + return std::static_pointer_cast(load(resolved_dictionary_name)); } ExternalDictionariesLoader::DictPtr ExternalDictionariesLoader::tryGetDictionary(const std::string & dictionary_name, ContextPtr local_context) const { std::string resolved_dictionary_name = resolveDictionaryName(dictionary_name, local_context->getCurrentDatabase()); + + if (local_context->hasQueryContext() && local_context->getSettingsRef().log_queries) + local_context->addQueryFactoriesInfo(Context::QueryLogFactories::Dictionary, resolved_dictionary_name); + return std::static_pointer_cast(tryLoad(resolved_dictionary_name)); } From bb323055c93bf8e2516533aad410e017b5b1250d Mon Sep 17 00:00:00 2001 From: Onehr7 <38950109+Onehr7@users.noreply.github.com> Date: Thu, 19 Aug 2021 11:47:30 +0800 Subject: [PATCH 201/220] Update troubleshooting.md --- docs/zh/operations/troubleshooting.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/zh/operations/troubleshooting.md b/docs/zh/operations/troubleshooting.md index 56b18aa1307..8d1defd6366 100644 --- a/docs/zh/operations/troubleshooting.md +++ b/docs/zh/operations/troubleshooting.md @@ -26,7 +26,7 @@ toc_title: "常见问题" ### 服务器未运行 {#server-is-not-running} -**检查服务器是否运行nnig** +**检查服务器是否正在运行** 命令: From 781b8123a591b4599b96cde27ab3693c7cf42761 Mon Sep 17 00:00:00 2001 From: Vladimir C Date: Thu, 19 Aug 2021 09:55:18 +0300 Subject: [PATCH 202/220] Remove logging from NotJoinedBlocks --- src/Interpreters/join_common.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Interpreters/join_common.cpp b/src/Interpreters/join_common.cpp index b230d8d1957..9890a130c33 100644 --- a/src/Interpreters/join_common.cpp +++ b/src/Interpreters/join_common.cpp @@ -500,8 +500,6 @@ NotJoinedBlocks::NotJoinedBlocks(std::unique_ptr filler_, , saved_block_sample(filler->getEmptyBlock()) , result_sample_block(materializeBlock(result_sample_block_)) { - LOG_DEBUG(&Poco::Logger::get("NotJoinedBlocks"), "saved_block_sample {}",saved_block_sample.dumpStructure()); - LOG_DEBUG(&Poco::Logger::get("NotJoinedBlocks"), "result_sample_block {}",result_sample_block.dumpStructure()); for (size_t left_pos = 0; left_pos < left_columns_count; ++left_pos) { From a27808be9846175222751a10771c8de6a6462b50 Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 19 Aug 2021 11:17:47 +0300 Subject: [PATCH 203/220] Revert "Do not miss exceptions from the ThreadPool" --- src/Common/ThreadPool.cpp | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 8ef85d82a1d..e6ccf405e9f 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -192,9 +192,6 @@ template ThreadPoolImpl::~ThreadPoolImpl() { finalize(); - /// wait() hadn't been called, log exception at least. - if (first_exception) - DB::tryLogException(first_exception, __PRETTY_FUNCTION__); } template @@ -273,21 +270,11 @@ void ThreadPoolImpl::worker(typename std::list::iterator thread_ } catch (...) { - ALLOW_ALLOCATIONS_IN_SCOPE; - /// job should be reset before decrementing scheduled_jobs to /// ensure that the Job destroyed before wait() returns. job = {}; { - /// In case thread pool will not be terminated on exception - /// (this is the case for GlobalThreadPool), - /// than first_exception may be overwritten and got lost, - /// and this usually is an error, since this will finish the thread, - /// and for this the caller may not be ready. - if (!shutdown_on_exception) - DB::tryLogException(std::current_exception(), __PRETTY_FUNCTION__); - std::unique_lock lock(mutex); if (!first_exception) first_exception = std::current_exception(); // NOLINT From df0303f9ba18a36eee4f87b0d6daf03b69b87f24 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 19 Aug 2021 11:19:23 +0300 Subject: [PATCH 204/220] Update filesystemHelpers.cpp --- src/Common/filesystemHelpers.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 5bed3ea1120..95913e6106c 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -124,8 +124,13 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p bool symlinkStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path) { + /// Differs from pathStartsWith in how `path` is normalized before comparison. + /// Make `path` absolute if it was relative and put it into normalized form: remove + /// `.` and `..` and extra `/`. Path is not canonized because otherwise path will + /// not be a path of a symlink itself. + auto absolute_path = std::filesystem::absolute(path); - absolute_path = absolute_path.lexically_normal(); + absolute_path = absolute_path.lexically_normal(); /// Normalize path. auto absolute_prefix_path = std::filesystem::weakly_canonical(prefix_path); auto [_, prefix_path_mismatch_it] = std::mismatch(absolute_path.begin(), absolute_path.end(), absolute_prefix_path.begin(), absolute_prefix_path.end()); From ba45bd834ff4a5e0d3c97c7f559522f9d2b92402 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 19 Aug 2021 11:22:57 +0300 Subject: [PATCH 205/220] Close #27816 --- src/Common/ThreadPool.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index e6ccf405e9f..4cb1df3ff65 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -191,6 +191,10 @@ void ThreadPoolImpl::wait() template ThreadPoolImpl::~ThreadPoolImpl() { + /// Note: should not use logger from here, + /// because it can be an instance of GlobalThreadPool that is a global variable + /// and the destruction order of global variables is unspecified. + finalize(); } From b03f851cba20ee54548d0067f8b5ab1733f67e9c Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 19 Aug 2021 11:39:50 +0300 Subject: [PATCH 206/220] Update filesystemHelpers.cpp --- src/Common/filesystemHelpers.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index 95913e6106c..d846f56c584 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -131,7 +131,8 @@ bool symlinkStartsWith(const std::filesystem::path & path, const std::filesystem auto absolute_path = std::filesystem::absolute(path); absolute_path = absolute_path.lexically_normal(); /// Normalize path. - auto absolute_prefix_path = std::filesystem::weakly_canonical(prefix_path); + auto absolute_prefix_path = std::filesystem::absolute(prefix_path); + absolute_pefix_path = absolute_prefix_path.lexically_normal(); /// Normalize path. auto [_, prefix_path_mismatch_it] = std::mismatch(absolute_path.begin(), absolute_path.end(), absolute_prefix_path.begin(), absolute_prefix_path.end()); From 57e817b71490c808e471705687c9104e8d8fbe1e Mon Sep 17 00:00:00 2001 From: alexey-milovidov Date: Thu, 19 Aug 2021 11:47:59 +0300 Subject: [PATCH 207/220] Update CHANGELOG.md --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 103d8e40fd9..71cdac17825 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ ### ClickHouse release v21.8, 2021-08-12 +#### Upgrade Notes +* New version is using `Map` data type for system logs tables (`system.query_log`, `system.query_thread_log`, `system.processes`, `system.opentelemetry_span_log`). These tables will be auto-created with new data types. Virtual columns are created to support old queries. Closes [#18698](https://github.com/ClickHouse/ClickHouse/issues/18698). [#23934](https://github.com/ClickHouse/ClickHouse/pull/23934), [#25773](https://github.com/ClickHouse/ClickHouse/pull/25773) ([hexiaoting](https://github.com/hexiaoting), [sundy-li](https://github.com/sundy-li), [Maksim Kita](https://github.com/kitaisreal)). If you want to *downgrade* from version 21.8 to older versions, you will need to cleanup system tables with logs manually. Look at `/var/lib/clickhouse/data/system/*_log`. + #### New Features * Add support for a part of SQL/JSON standard. [#24148](https://github.com/ClickHouse/ClickHouse/pull/24148) ([l1tsolaiki](https://github.com/l1tsolaiki), [Kseniia Sumarokova](https://github.com/kssenii)). From c68793cd1291d9cb39ff76cde17ab1a8779ec4c3 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 19 Aug 2021 12:14:41 +0300 Subject: [PATCH 208/220] Update filesystemHelpers.cpp --- src/Common/filesystemHelpers.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Common/filesystemHelpers.cpp b/src/Common/filesystemHelpers.cpp index d846f56c584..9c3db0f3e30 100644 --- a/src/Common/filesystemHelpers.cpp +++ b/src/Common/filesystemHelpers.cpp @@ -132,7 +132,7 @@ bool symlinkStartsWith(const std::filesystem::path & path, const std::filesystem auto absolute_path = std::filesystem::absolute(path); absolute_path = absolute_path.lexically_normal(); /// Normalize path. auto absolute_prefix_path = std::filesystem::absolute(prefix_path); - absolute_pefix_path = absolute_prefix_path.lexically_normal(); /// Normalize path. + absolute_prefix_path = absolute_prefix_path.lexically_normal(); /// Normalize path. auto [_, prefix_path_mismatch_it] = std::mismatch(absolute_path.begin(), absolute_path.end(), absolute_prefix_path.begin(), absolute_prefix_path.end()); From 3d05014da1d191a389a92a5818c062696fc151e8 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 19 Aug 2021 12:54:03 +0300 Subject: [PATCH 209/220] Update FunctionSnowflake.h --- src/Functions/FunctionSnowflake.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Functions/FunctionSnowflake.h b/src/Functions/FunctionSnowflake.h index 3f0f404f7e4..95d02de3a2b 100644 --- a/src/Functions/FunctionSnowflake.h +++ b/src/Functions/FunctionSnowflake.h @@ -39,6 +39,7 @@ public: size_t getNumberOfArguments() const override { return 1; } bool isVariadic() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -79,6 +80,7 @@ public: size_t getNumberOfArguments() const override { return 0; } bool isVariadic() const override { return true; } bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -126,6 +128,7 @@ public: size_t getNumberOfArguments() const override { return 1; } bool isVariadic() const override { return false; } bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { @@ -166,6 +169,7 @@ public: size_t getNumberOfArguments() const override { return 0; } bool isVariadic() const override { return true; } bool useDefaultImplementationForConstants() const override { return true; } + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override { From f94b0b8d3f81f2bebf7051530d6b31dae5be73bc Mon Sep 17 00:00:00 2001 From: tavplubix Date: Thu, 19 Aug 2021 13:19:48 +0300 Subject: [PATCH 210/220] Update DDLWorker.cpp --- src/Interpreters/DDLWorker.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 856d8713560..c2de6ecbaf1 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -634,7 +634,7 @@ void DDLWorker::processTask(DDLTaskBase & task, const ZooKeeperPtr & zookeeper) String dummy; if (zookeeper->tryGet(active_node_path, dummy, nullptr, eph_node_disappeared)) { - constexpr int timeout_ms = 30 * 1000; + constexpr int timeout_ms = 60 * 1000; if (!eph_node_disappeared->tryWait(timeout_ms)) throw Exception(ErrorCodes::LOGICAL_ERROR, "Ephemeral node {} still exists, " "probably it's owned by someone else", active_node_path); From 51d802b82516f27ee345360c30fa20b8703bb484 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 19 Aug 2021 14:13:03 +0300 Subject: [PATCH 211/220] try to update version to 21.10 because 21.9 has a broken release PR and branch --- cmake/autogenerated_versions.txt | 10 +++++----- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 2435335f669..03247b4b3ea 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54454) +SET(VERSION_REVISION 54455) SET(VERSION_MAJOR 21) -SET(VERSION_MINOR 9) +SET(VERSION_MINOR 10) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH f063e44131a048ba2d9af8075f03700fd5ec3e69) -SET(VERSION_DESCRIBE v21.9.1.7770-prestable) -SET(VERSION_STRING 21.9.1.7770) +SET(VERSION_GITHASH 09df5018f95edcd0f759d4689ac5d029dd400c2a) +SET(VERSION_DESCRIBE v21.10.1.1-testing) +SET(VERSION_STRING 21.10.1.1) # end of autochange diff --git a/debian/changelog b/debian/changelog index 38f740ae062..f3e740d20cf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (21.9.1.1) unstable; urgency=low +clickhouse (21.10.1.1) unstable; urgency=low * Modified source code - -- clickhouse-release Sat, 10 Jul 2021 08:22:49 +0300 + -- clickhouse-release Sat, 17 Jul 2021 08:45:03 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index f17fa8ade16..052e008fd56 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 5da9e703f4d..25f01230c5f 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* ARG gosu_ver=1.10 # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 5768753cd7c..62cfcf9e896 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From fce6eed2be50def40abdffdb646381d9ad7e4378 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 19 Aug 2021 14:21:36 +0300 Subject: [PATCH 212/220] Revert "try to update version to 21.10 because 21.9 has a broken release PR and" This reverts commit 51d802b82516f27ee345360c30fa20b8703bb484. --- cmake/autogenerated_versions.txt | 10 +++++----- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 03247b4b3ea..2435335f669 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54455) +SET(VERSION_REVISION 54454) SET(VERSION_MAJOR 21) -SET(VERSION_MINOR 10) +SET(VERSION_MINOR 9) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH 09df5018f95edcd0f759d4689ac5d029dd400c2a) -SET(VERSION_DESCRIBE v21.10.1.1-testing) -SET(VERSION_STRING 21.10.1.1) +SET(VERSION_GITHASH f063e44131a048ba2d9af8075f03700fd5ec3e69) +SET(VERSION_DESCRIBE v21.9.1.7770-prestable) +SET(VERSION_STRING 21.9.1.7770) # end of autochange diff --git a/debian/changelog b/debian/changelog index f3e740d20cf..38f740ae062 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (21.10.1.1) unstable; urgency=low +clickhouse (21.9.1.1) unstable; urgency=low * Modified source code - -- clickhouse-release Sat, 17 Jul 2021 08:45:03 +0300 + -- clickhouse-release Sat, 10 Jul 2021 08:22:49 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index 052e008fd56..f17fa8ade16 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.10.1.* +ARG version=21.9.1.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 25f01230c5f..5da9e703f4d 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.10.1.* +ARG version=21.9.1.* ARG gosu_ver=1.10 # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 62cfcf9e896..5768753cd7c 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.10.1.* +ARG version=21.9.1.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From c5fb6b3670a4e6d74ca795ca87e847f7acce7e39 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 19 Aug 2021 14:27:49 +0300 Subject: [PATCH 213/220] Revert "Revert "try to update version to 21.10 because 21.9 has a broken release PR and"" This reverts commit fce6eed2be50def40abdffdb646381d9ad7e4378. --- cmake/autogenerated_versions.txt | 10 +++++----- debian/changelog | 4 ++-- docker/client/Dockerfile | 2 +- docker/server/Dockerfile | 2 +- docker/test/Dockerfile | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 2435335f669..03247b4b3ea 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54454) +SET(VERSION_REVISION 54455) SET(VERSION_MAJOR 21) -SET(VERSION_MINOR 9) +SET(VERSION_MINOR 10) SET(VERSION_PATCH 1) -SET(VERSION_GITHASH f063e44131a048ba2d9af8075f03700fd5ec3e69) -SET(VERSION_DESCRIBE v21.9.1.7770-prestable) -SET(VERSION_STRING 21.9.1.7770) +SET(VERSION_GITHASH 09df5018f95edcd0f759d4689ac5d029dd400c2a) +SET(VERSION_DESCRIBE v21.10.1.1-testing) +SET(VERSION_STRING 21.10.1.1) # end of autochange diff --git a/debian/changelog b/debian/changelog index 38f740ae062..f3e740d20cf 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,5 +1,5 @@ -clickhouse (21.9.1.1) unstable; urgency=low +clickhouse (21.10.1.1) unstable; urgency=low * Modified source code - -- clickhouse-release Sat, 10 Jul 2021 08:22:49 +0300 + -- clickhouse-release Sat, 17 Jul 2021 08:45:03 +0300 diff --git a/docker/client/Dockerfile b/docker/client/Dockerfile index f17fa8ade16..052e008fd56 100644 --- a/docker/client/Dockerfile +++ b/docker/client/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* RUN apt-get update \ && apt-get install --yes --no-install-recommends \ diff --git a/docker/server/Dockerfile b/docker/server/Dockerfile index 5da9e703f4d..25f01230c5f 100644 --- a/docker/server/Dockerfile +++ b/docker/server/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:20.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* ARG gosu_ver=1.10 # set non-empty deb_location_url url to create a docker image diff --git a/docker/test/Dockerfile b/docker/test/Dockerfile index 5768753cd7c..62cfcf9e896 100644 --- a/docker/test/Dockerfile +++ b/docker/test/Dockerfile @@ -1,7 +1,7 @@ FROM ubuntu:18.04 ARG repository="deb https://repo.clickhouse.tech/deb/stable/ main/" -ARG version=21.9.1.* +ARG version=21.10.1.* RUN apt-get update && \ apt-get install -y apt-transport-https dirmngr && \ From 1697a6fa22d239a570f0570762c864cb6b09f8a3 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 19 Aug 2021 15:10:55 +0300 Subject: [PATCH 214/220] Fix --- src/Core/PostgreSQL/insertPostgreSQLValue.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp index 19560cec9ea..4f31c8dfb52 100644 --- a/src/Core/PostgreSQL/insertPostgreSQLValue.cpp +++ b/src/Core/PostgreSQL/insertPostgreSQLValue.cpp @@ -110,7 +110,7 @@ void insertPostgreSQLValue( readDateTime64Text(time, 6, in, assert_cast(data_type.get())->getTimeZone()); if (time < 0) time = 0; - assert_cast &>(column).insertValue(time); + assert_cast(column).insertValue(time); break; } case ExternalResultDescription::ValueType::vtDecimal32: [[fallthrough]]; From a9d83c1eec7fd5723a29d0182929d73970a8b713 Mon Sep 17 00:00:00 2001 From: Anton Popov Date: Thu, 19 Aug 2021 18:15:44 +0300 Subject: [PATCH 215/220] fix postgres like cast with negative numbers --- src/Parsers/ExpressionElementParsers.cpp | 30 ++++++++++++++----- src/Parsers/ExpressionListParsers.cpp | 6 ++-- .../01852_cast_operator_3.reference | 10 +++++++ .../0_stateless/01852_cast_operator_3.sql | 14 +++++++++ .../01852_cast_operator_bad_cases.reference | 8 +++++ .../01852_cast_operator_bad_cases.sh | 10 +++++++ 6 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/01852_cast_operator_3.reference create mode 100644 tests/queries/0_stateless/01852_cast_operator_3.sql diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 16f2b720b4a..a79b3e51e16 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -850,15 +850,24 @@ static bool isOneOf(TokenType token) return ((token == tokens) || ...); } - bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { - /// Parse numbers (including decimals), strings and arrays of them. + /// Parse numbers (including decimals), strings, arrays and tuples of them. const char * data_begin = pos->begin; const char * data_end = pos->end; bool is_string_literal = pos->type == TokenType::StringLiteral; - if (pos->type == TokenType::Number || is_string_literal) + + if (pos->type == TokenType::Minus) + { + ++pos; + if (pos->type != TokenType::Number) + return false; + + data_end = pos->end; + ++pos; + } + else if (pos->type == TokenType::Number || is_string_literal) { ++pos; } @@ -876,7 +885,7 @@ bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } else if (pos->type == TokenType::ClosingSquareBracket) { - if (isOneOf(last_token)) + if (isOneOf(last_token)) return false; if (stack.empty() || stack.back() != TokenType::OpeningSquareBracket) return false; @@ -884,7 +893,7 @@ bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } else if (pos->type == TokenType::ClosingRoundBracket) { - if (isOneOf(last_token)) + if (isOneOf(last_token)) return false; if (stack.empty() || stack.back() != TokenType::OpeningRoundBracket) return false; @@ -892,10 +901,15 @@ bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected } else if (pos->type == TokenType::Comma) { - if (isOneOf(last_token)) + if (isOneOf(last_token)) return false; } - else if (isOneOf(pos->type)) + else if (pos->type == TokenType::Number) + { + if (!isOneOf(last_token)) + return false; + } + else if (isOneOf(pos->type)) { if (!isOneOf(last_token)) return false; @@ -915,6 +929,8 @@ bool ParserCastOperator::parseImpl(Pos & pos, ASTPtr & node, Expected & expected if (!stack.empty()) return false; } + else + return false; ASTPtr type_ast; if (ParserToken(TokenType::DoubleColon).ignore(pos, expected) diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 58f5e766905..3aa5c82884b 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -664,10 +664,12 @@ bool ParserUnaryExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (pos->type == TokenType::Minus) { - ParserLiteral lit_p; Pos begin = pos; + if (ParserCastOperator().parse(pos, node, expected)) + return true; - if (lit_p.parse(pos, node, expected)) + pos = begin; + if (ParserLiteral().parse(pos, node, expected)) return true; pos = begin; diff --git a/tests/queries/0_stateless/01852_cast_operator_3.reference b/tests/queries/0_stateless/01852_cast_operator_3.reference new file mode 100644 index 00000000000..a1e54797d60 --- /dev/null +++ b/tests/queries/0_stateless/01852_cast_operator_3.reference @@ -0,0 +1,10 @@ +-1 +SELECT CAST(\'-1\', \'Int32\') +-0.1 +SELECT CAST(\'-0.1\', \'Decimal(38, 38)\') +-0.111 +SELECT CAST(\'-0.111\', \'Float64\') +[-1,2,-3] +SELECT CAST(\'[-1, 2, -3]\', \'Array(Int32)\') +[-1.1,2,-3] +SELECT CAST(\'[-1.1, 2, -3]\', \'Array(Float64)\') diff --git a/tests/queries/0_stateless/01852_cast_operator_3.sql b/tests/queries/0_stateless/01852_cast_operator_3.sql new file mode 100644 index 00000000000..1ad015a8dc4 --- /dev/null +++ b/tests/queries/0_stateless/01852_cast_operator_3.sql @@ -0,0 +1,14 @@ +SELECT -1::Int32; +EXPLAIN SYNTAX SELECT -1::Int32; + +SELECT -0.1::Decimal(38, 38); +EXPLAIN SYNTAX SELECT -0.1::Decimal(38, 38); + +SELECT -0.111::Float64; +EXPLAIN SYNTAX SELECT -0.111::Float64; + +SELECT [-1, 2, -3]::Array(Int32); +EXPLAIN SYNTAX SELECT [-1, 2, -3]::Array(Int32); + +SELECT [-1.1, 2, -3]::Array(Float64); +EXPLAIN SYNTAX SELECT [-1.1, 2, -3]::Array(Float64); diff --git a/tests/queries/0_stateless/01852_cast_operator_bad_cases.reference b/tests/queries/0_stateless/01852_cast_operator_bad_cases.reference index 2c4517e0eda..b179e5e927a 100644 --- a/tests/queries/0_stateless/01852_cast_operator_bad_cases.reference +++ b/tests/queries/0_stateless/01852_cast_operator_bad_cases.reference @@ -8,3 +8,11 @@ Syntax error Syntax error Syntax error Code: 6 +Syntax error +Syntax error +Syntax error +Syntax error +Syntax error +Syntax error +Syntax error +Syntax error diff --git a/tests/queries/0_stateless/01852_cast_operator_bad_cases.sh b/tests/queries/0_stateless/01852_cast_operator_bad_cases.sh index f2f566b78c4..6c578a0996c 100755 --- a/tests/queries/0_stateless/01852_cast_operator_bad_cases.sh +++ b/tests/queries/0_stateless/01852_cast_operator_bad_cases.sh @@ -15,3 +15,13 @@ $CLICKHOUSE_CLIENT --query="SELECT [1 2]::Array(UInt8)" 2>&1 | grep -o -m1 'Syn $CLICKHOUSE_CLIENT --query="SELECT 1 4::UInt32" 2>&1 | grep -o 'Syntax error' $CLICKHOUSE_CLIENT --query="SELECT '1' '4'::UInt32" 2>&1 | grep -o -m1 'Syntax error' $CLICKHOUSE_CLIENT --query="SELECT '1''4'::UInt32" 2>&1 | grep -o -m1 'Code: 6' + +$CLICKHOUSE_CLIENT --query="SELECT ::UInt32" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT ::String" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT -::Int32" 2>&1 | grep -o 'Syntax error' + +$CLICKHOUSE_CLIENT --query="SELECT [1, -]::Array(Int32)" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT [1, 3-]::Array(Int32)" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT [-, 2]::Array(Int32)" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT [--, 2]::Array(Int32)" 2>&1 | grep -o 'Syntax error' +$CLICKHOUSE_CLIENT --query="SELECT [1, 2]-::Array(Int32)" 2>&1 | grep -o 'Syntax error' From f3ff3aee0e749e3e8242ebedeba0e867519b2b56 Mon Sep 17 00:00:00 2001 From: Pavel Kruglov Date: Thu, 19 Aug 2021 18:49:39 +0300 Subject: [PATCH 216/220] Remove tmp folders from tests --- .../test_abxi8n/parquet_decimal0.parquet | Bin 8849 -> 0 bytes .../test_abxi8n/parquet_decimal1.parquet | Bin 29278 -> 0 bytes .../test_abxi8n/parquet_decimal2.parquet | Bin 6038 -> 0 bytes .../test_abxi8n/parquet_decimal3_1.parquet | Bin 559 -> 0 bytes .../test_abxi8n/parquet_decimal3_2.parquet | Bin 777 -> 0 bytes .../test_abxi8n/parquet_decimal3_3.parquet | Bin 3049 -> 0 bytes .../0_stateless/test_dozlem/arrays.arrow | Bin 4834 -> 0 bytes .../test_dozlem/arrow_all_types_1.arrow | Bin 3600 -> 0 bytes .../test_dozlem/arrow_all_types_2.arrow | Bin 1872 -> 0 bytes .../test_dozlem/arrow_all_types_5.arrow | Bin 1816 -> 0 bytes .../0_stateless/test_dozlem/dicts.arrow | Bin 20030554 -> 0 bytes .../test_dozlem/nullable_arrays.arrow | Bin 1322 -> 0 bytes 12 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/queries/0_stateless/test_abxi8n/parquet_decimal0.parquet delete mode 100644 tests/queries/0_stateless/test_abxi8n/parquet_decimal1.parquet delete mode 100644 tests/queries/0_stateless/test_abxi8n/parquet_decimal2.parquet delete mode 100644 tests/queries/0_stateless/test_abxi8n/parquet_decimal3_1.parquet delete mode 100644 tests/queries/0_stateless/test_abxi8n/parquet_decimal3_2.parquet delete mode 100644 tests/queries/0_stateless/test_abxi8n/parquet_decimal3_3.parquet delete mode 100644 tests/queries/0_stateless/test_dozlem/arrays.arrow delete mode 100644 tests/queries/0_stateless/test_dozlem/arrow_all_types_1.arrow delete mode 100644 tests/queries/0_stateless/test_dozlem/arrow_all_types_2.arrow delete mode 100644 tests/queries/0_stateless/test_dozlem/arrow_all_types_5.arrow delete mode 100644 tests/queries/0_stateless/test_dozlem/dicts.arrow delete mode 100644 tests/queries/0_stateless/test_dozlem/nullable_arrays.arrow diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal0.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal0.parquet deleted file mode 100644 index cfa6cc2e0b68618e3e064d1825c6aa83dddc7be2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8849 zcmdT~U1$_n6uvXA+jSF{WxIE>OID3xJ*I7&NFF2v5s^|v#D`Ey2qE;L4J~iB!bG zk(hIa3C6 z?I;@|tyB}5c_O5xDwwXScp**1NqT)mNQ*HTi&cb>CO;%8TqQz!lYmEwvJuj4gV5|J zLRw40>tw|X=~GA-LWHOKx0 zJq7B}9NtT4QbS$c1F!bD^4&b5l6E0OmFaJ^N=>pikxmhJIq5yklJuyd{#y@I^|E|R zpHZ2<7l!&`KS_Za>H@$MD4RNX_hUkn8tRW9!F+=&q|JZpDgTp&ge3p4+$a$y2}x9q zBJOg8^rnfVM-BB(BfR=V7EiuJ|-0I4A^N32?h3cS8e`AG< zN_9}AQ^Z|PdJAnNJ!+`;PQct(vV2RQQJKCMb?~24Bn4`yw@<1A)ezR z4vr+wRm3q8;y98x{bqa-C-I*+17>^?Cu_&i1DKhPhaGG1v#$(6 zc6Gtaek}OeHH9F1mf&T#5&Y~4LXaIf*jj7NSlnsoxNME7SduYZ!Cp4Wz|U3|1ldRe zFIzz1XQKv!Y?FYO4G!?L4FN$m0bpypd20a$|y6|;d=Y_K!7t`bT7Ds_UHj1yEbmGe>?*Z@DRF41v diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal1.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal1.parquet deleted file mode 100644 index 19d5e387f8da6bc9675ea6fa63d2f021662c3e1f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 29278 zcmdU24{#LK8Q-@#XrLk9(Cqf;q#6}J(Ln~RSQTtzYg_-MlomT!br|_4pw;3()LO@w z#uOt)M2s;;h%{nMuQWwOs*%YsDaABmN-2X%8HQ4d7*mQ&hzw);eYf}azP)|N-Eq5Y zxMA|{-oAbB_wD<=?|a|A_dTXw^^*xk$Os!nWtfCUYFd7A$J|Rr=gnW$49ys73?EDR z_+@BuBt*H95M4L5G3=M7X?gh~PlpCkY!r@~7}AE0iinwpZidVy!dx!kGRQyC{iCc_ zbZRqX=vW^nCB|q8Dm?k{S=W}`*r18eDr!2`171%CmoKkG@?}{BsmQg#_T@3KvtnS| zgxMkBunueW;)vqj6g4X0mm1?F{1Q#OLTHoQu6ud@!_z8e9KF5hRz%^JAxyda6A35Z zg9tJxg>_gRvhhny#@R6$8+5Zqhhsyabx3g}gSs+oQTYs9yc!r7IO^)U5alL#mH1_g z(KVcjF^BQ$z%bY_44IX<2jQ|`ni7_SS@L;f74&h#%a7@J?Ef+{%YAQ(8kO+f%!=cZiCYFmKg+^QrVL8iG9C^y8E<3ET5|^MD^xP;BwJKI z0~fRCk-C{hxd~<^e&IK3fRq<8G3GI5HGBnnMk2Ek_YkuZztqf%!P4W3lnT?w1&j5+ zRYKF8{V#JdMVKKCwwes){20urFxLs_G@#yaFiPH=qDJM@j1342c6hR^K&CGE3Yy5W zGR4L-D21gh@s@mziMJpo-f>}8>aZmOiz7-h$(CfAVT(%fbz^9mVTmsk-6gb$pbm-jLq2Fk5;wS%8&hrr*gUDV_%92D zuE7^)G7w~Nb+b~S&4v0i;qccj#i*`93CTfUYK}fGSW*oR*b0&0DROO?U1u|xWNLNO zqVJXxs4GGGS}mb?_O}!@Dq%b5E31;J3w~H$k!i%0gN?iL+@);ss?THMk(Jd=i@uxB zg{pIrzOp5mx^psYQ7OI`L|QtAi9i-s5-EBOmB*VH^{WjH)Ek&aqH8)^|PV1(8-wU?Pw~ zl|EJ|Yndm!_90t{GK*T^qkjaRpHu-C>V3^6E>LwY~&dZ_sax|!pd(!Jcdf4(o zCp=inwdQo#e0(-XdHAtrB8`(72+M=D3~JjXSU$;~57G@uH7FrD3@Y739~Z1F^&rj% zd%wkCTE44gP!C=SZ(W(Q9xOGkwtd%Mt?PJw`YR7jdc5!cf3_ofoFO?sE^!oePOd{u z=j2k@4)zMZSLpf<6W4ZQn{-%w6*QMB>6~0whAk?4(V-Vfc$dhGN=dYk#M<^fCW7U{ zTEyzT8a7{@HL;W~tbFm|g2xCZ%OFfeuKmPYuVHYx3mg3b^nBkzurxI)pQvNTMo*M~ zT<{fy;^pbPx_gsL;csi0_g>G$v%FVp#@2Ej9K2RZsN}jbY*8t`7lb-8g^6H!t`?z| zPJshI9CU=*_G1Ru@?fpEZCj^8&D7JLP)mQx#N!@p*ECo%&38gM46a)}IlWOi2>Pcp z%n1)RIvuK~XYaw%_>d#lo+78wi#X-2xQT(_9_)&rLEnv8)A)o2*M`M3pB&aseJFD= zwPxnsw=kI8gYCK*8g6#bSKgbVMx{c?NFUnpJ=TZV3pKYg@wf-uIRmO^_)g!H`p_0R zJ&~5rW+HG8wsjUPo|PMsqzW5;9fTn3YSu)F8LnmI@$*;e8k5y_paegWwq zlDtW#F8JQPn2_V7aRC#Ld$7CaLBl*GlEflYcMcg=c~z$FQfl$NAkyleGZDB4yXg+7 zzauvyN!{S;iR?Afk-Hd7?!g|q6WZ@Q?KRTcdzg6KgY8`iYZm&hk<W!EU_|qW7ILgWB`}1Hlby`y!}YG-&91_#p<98`RzhVdI0RJ$+jrVd8Ov+Ve0( zANHNTUg*K~<$N`tThwijLd&CrhDlYw<~X@Yt*8*N?J?mknUY^DxKw#KxkSX3LnbbL zDSPtSS;ZyfR<-JJ0h=pP!VId*uuxRALRrAR%?Q=;-~vfr|V zBjvVr^^*ensy$|uoD|5^%_wRfqMrtVUB#d2qQpXL&B^V(7);NVllC|Qk6 z-8mUHs_eUyGO)Xrao7n9yL6d=-e(ZB(=IC;PD$kgd)bvFPW_g+K!cv;90wWMg7~`Y zV%+Ss=UI=bY$MBZmWW8(pXW%)&KAU%UfXH~EU7(3RJJAXj#V)#qlA?&atX=Q7Q|Oy z+g}h+{(}Eh_JUDMS8yrF*h)r4UlLIH(%@m#zDACe%&lb9o|OVxSDyZiYHs2ZlDU

hQR`miQjocojOt!3pn3J+VbtOlj+D%;WK`vA0=75j&Zt|BDP_2&hW59P zFHN3!4ba|9Y2;*zHW-KXSmQ*BE4d=IIZQ5vErbg6yr$vM|9dVs8Q}tRqi(k1_S2|< zEx%LqEV;f6TUFXZwGp;uc3M-ZaQC-z*kptYVhgqPtpfW0fS{eXDH~3yGHnk#^;A&K zVfxl`9Ng1gwif3wfApB(HnJ>@FuoV)e1jw9&hBR1!dJU4cY-@!`$-PV^c2h|YyQL~ zE(P~^*S{^G=Iz15sNRhnDff8~z9ZnRcTRss zt=+^WN0FpsiV~9);26oyD4FJY~`>MK5zef z0(QKIpq;lV8%}9%v)CE6y_@48a~q2LyocV$*~**SO^)wPQwC37nowdErLbsy!s9}<*@2=zD>)ZQbQ(am0Uti ztMsKT4T;i2Ll$>xSkD_m^HZhTp^n`o83I?|BoOJ%u!YfGKRFTmly`d?fMvbmC0 zoAzR0#NNR}t7HG>NXh0(TJ`P2_xk(1pcUK5ncOM%ggXA^6fkV}$6R8v!7+y6PR{xR zIB)%r{|r;xV4Ge}V;Zx;NU5$K4x4OnDCY8(_u$m%69nzNEjx{A+y6AwMn`g2FUP@M z-u_PobbsnGwQXcs+F(T5`ah18yS&ju0*?PTcWMXi@*X+DCFCw|>t_NE9VRL#k1%(R zVmhg?)LAM^yWi3?s=S{|!Cl^lqc~UU^M+CG>Ahr>u!CV~I z3uyeDF)Dqf$`N`ZM!6+O?%b3273566?h7s9^msPYM<^b@jRRMTTR z+xoy=-u7}G>dG;)J~b_ssQg;y?f*4L%3aW+@GX^rf(J zT5B1d2i?zd>8(}bJ-XSZL-jHpmOn!yA$@%rwyJzOFaEOKWA(-jL!svd z4&7Qq-YU!;0`}MGP+ONf7fX$B8SdvGDwb&4B7D#UJa(Rqy}XsI`i7}##-+#aic*-1 zuNd>f(C`wM%UU@`2O4L*J73fx`XaF`eRagL^rh4qMa?o%weqA`wzL2?HFA-xWn_zG zyH?_XKP&xV8BMpG10q|N%^3n4e#@a->&h0()~?cF@5{NfEF=4#l5aWMcij7Od^MNQ zT0yqr*fzX^I=xB3x07#FsntS8z8q(INV8|jw#-7Hu|`72()c(DLCk-&_cd zujz2S8BsfLQ?#s)YMZDh@D;c+r#`Y<w!fO^VkIl^3Vgw?eqe(h3s(SWb;^IQHaK{m*)_G zpQi-`8mkXpM%ja(vG5RNBszE*Lk@mMdP9(L);R`DTXwkOCbGjrX$*TA`V4*sC_|9p z!Qg3NFZdjO3jqheg2!R2;B#;(1RUB4Imc*P!@B7K8rJzVtUZpl^gAXp=!it{IF=B6 zj`)Ls+Obq?*R;Y2VU+w z>CEmh`roILMHEE3)5VQ}14{IB&`s9OYy3G$XuC0pC zC67g6@uv?WMSPGDq>V+S1noocp%m$Zh9V_|KC~jz1}O?!jpy9ixpQXk-Da`7?2_EQ z^Ub~A`A+UP=g!U8ftQ9I+lf0pMaOdNln{2$#>U2qBb-jByFeIrICy^PkMQ6r~jXRcWxpOi1bPR?de!uyd0>-c%OJGd;67-dH(t)6O7DLlV z7Zvw!pfJ@#ajGApO62MmCh~|Bxe#+N#NcNO=B;cvBka{-G|J97`9OB^nX0(pOP8Mv zZmBdh1cPgsry(l6Pab0FO~~}_#oRdyu64kfj;uaYM;eT1Oh00yn|~gnC9Sa$p`J}G z29n?Yl}h2WZkEDPW}|Pr;G-@yQe6h4q==FJ8n`|>|0)`+R#rT?pe_n zi5cm)t+2Eejik@5!)TNpG1A(k*LU7tDS!C!t=Cb2EDNHgWT}{CkDXK{I7>tr4 zM#2a7kG{%)z7kY>Q?1+)7b>^S1Uc7N> z|80|zYDcL0pytrBZazw-cPYctn`TD3w+)uIg%wHh|L=&#^l@to3bJM2nbmA$-I?|9 z7n7>?l<>tY6Fwt_m)&<^bvs<#o~1c8^NismK1S+2ngE+nPqyUa@Jb0;-ob?MeRZuD zzUf6DhjVHejFKWg#;NqtWySp)C{6W{T=hc~$NAfTXDIjRLT5&50lAv*0Z}| zy&sLF&#l8~lpQfr-+qjmKIQZJd?E}JPpC%8m5m@>D?G)>G7TQ(*mpx zpplvu(U^Y3NaDGRV_M2Kp;%S0B^!iSDurK$SPFb+og0GdPf$gw$6%BcF%kypT08!e zz7kYFQ?1+&tD1EHrV(+t8@D!!}RfGDV zMyQIR5HgZa6Y}VzaD-ox?morR<2&oQ5mpVFzRXRTIq=l2fBdow<=VRR{Bvp8gS`3JI zlJFxE&r3Y-<2k}7Bwmzw(Z?mvQN&Y{2V2HdLOkyf54Mcw7~(~Tc(7$WlZYb|;@C2t zDalVfsR+-s5| znt5D;Rvv1gokt3I@MD2{_hHw?9 zX1<(3D_^0Yoi8x(;4dNmbG*ERo?}b&?<|j>c=PSjTZ0qja_*hdiIdZB%;tvoj_e&C cJbduL*x?rry_{Ld3UQwwaSiwOG5nAGe^7mF-2eap diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_1.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_1.parquet deleted file mode 100644 index 39acd4a8c14b4c0432621d7fcfbe8f4bc13a2c17..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 559 zcmWG=3^EjD5oHi%@&OVoLJSNHYHBi~e3GUt3~Z7TjESO5Vhmz3VsdH>Y8;v}K&33A zOh85GdiofVG$kQvD!`-Z2ePJQBu%#%@#uQOgk)6;lCEP+$hw#$V@w$&*ou=gQgahU zIT%FwL^(v6fId>?0{T=)Ck8}rXeRUqODAn|0dxDQy|2Ovt*%!*ZrFfeE+6(klFmZp~ICKnVal%*CGXXfWA i80wko8R`c6IR*rK_=iaP$S^R#5rZEi1495XsQ>`uct6?z diff --git a/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_2.parquet b/tests/queries/0_stateless/test_abxi8n/parquet_decimal3_2.parquet deleted file mode 100644 index 0de83a7eeb6422516f92630c7129dc558c5143c9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 777 zcmWG=3^EjD5oHi%@&OVoLJSNHK$=ffNJo?jD9$FzCS%3~6l9kGkwC1LBqPcvY0ARD zCMm&~D9R+pBo-nTrKYCF0a7Tb1JuMK$^ljA6vxjO*Kui@xFL77TMrQmPaV>V3}IsF#$Z82JE1OV8VLpV&l6ClP+9Xj#OZsHThh$BWsk_dz z?`d!c%->O*GLED(N^oS+5YZ4uJ#mfG5_*>9!x`NbebTl+G9l1ON#h>+jvFrhG@ zF7{oNKh`CFW6tmDaG6Si-!UZq(VRb7g27S}{Bc_12XPVW(@n#r5zDt&VP*@{#pM>i zs5Q*s2HInOh8CHhhbs1yt#Y|6O(_W3SB;-(@Xdq)n&lUS81ACMZD*vJguxxHKz}86 zaTLsXzAEv<;GV9+G?N6sos;-sa7S6_XOrNcTM|F~xK1sY=3@CpR+^c|x_51nSravp zHM@1)YIa1y;77D*FMM7$+1X8L>OpAVVM?MfU4#Cb8+|^i{NcEZny|~InT960U@&B{ zi>zS6$-2Z3P10M3E1m?u>qz|2Bo{Ue?Iie<4T&Fqq`eKea+2V8U5PKP|0WDKWBAlL zs1n;aX*_8@t20wnD3ejFd$(_3xyqLh#1RdNx+U~OUoZK3DbR~TZ}__B>t3MyoTHyO z+;lw0OR&cNMcD-PN(k!3jlWNEiZ9APrwWANJFC|(_?Pts)Jq1aR{{P-tim|WN2vmC zVLVVz^q`)*;eX~#P)~#Wt&XeI&hZa=K^Lo}p6Nh6d4YO90`(Nb$GC&W7vkS(!5~|= zTJ__XubZ#78?9FHP4oEV>5J#Z%1(8svc30Ux3+ix;lAVHPHp}tpKz;Q!%hAVm)RJU diff --git a/tests/queries/0_stateless/test_dozlem/arrays.arrow b/tests/queries/0_stateless/test_dozlem/arrays.arrow deleted file mode 100644 index 9ab37ff58909f6b01e1336deecf64e1a08f073b6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4834 zcmeHLK}b|V82;b7+G1NQBBCI%f`=_Oc$(E_61zpL%EK#wCg*KfbKnX=@R0$?qsuk-!N)a1;^V8K_hhaN^4aV8t)6tH<5=@CB((! zp*VD-uDCL1pwxUg^H7CnQ-vR!Me8NRwxP^Gx9r6B78>(UXxe`7KM(vo@Z8Su-J78c zDy>y8<|Syxi^q&`%mj3CCuYI5@Ew{c^y|grp60k2=$1=}JMYA^ICjTw(Rnz8!}S4`-2yuIIe$kG{XBNg zNg>_loP;L&`6vEWhQIT-1(gmLT}z(w;a)y?@O@#Hu2uehg~Z4CkIheYIs^OF=%~W0 za^EHoNLe+oALDRx^gQw|&lgf|&f0UWe;b_7+1STzf)P#`=UcZ47Jr@_w+R-1+fAY^ zSp1^~FWAXpSX%}!Sp0hpUagAly>`{+O&M5#k*Lb)LD7;5(SPQHb2B^%fubn-5e!5}x!W;YZxqSTWgQ1s88x67Tw+|N% zJ>H5rnu}4c-Elq~I~#tkr^?;m-7p;Ke%jTZS+P6V2UEr80QKJV<894f+QnM(ifC*PTB L1_>ml{6y%82!?uNr*u+6bT|FgF^?Usg{yKC5S_Da_o>Hg;o=&6>NlV?a-k^2f+@G zox~p@D8<21oU-WbB#2AE>hqj??}^VFLw3nm?)&cf&UxRx@80{p*Ca`j6tiX(qm`^| zd7HH&daz6m*no<-=EBwNJkZIESqp6eZICg0hjxtCMysO@pm~q>0(dT(XfJ|ZU`%MMf88hFijsJ(F=(LA0v zk3U+*Ui*98nxylSf7;zOZ9P>2pFo*QDXrFz42gr?X;n?hF;& ziLHe5?4P>^Ui(1rZig>ITeO0^4=}?#7&fV*9_G-emF*(NU`3Q&X3OT>894Mbt|j0y z^81=O!`bKj;f6Lp?2UG$# zFD$+p@xsc#8u7xq|9->^i(d=8a68fqi|+*fLh!qhURdk5AMwJPKWV-fwO_=v{|)cE zr%GK$0yY}4Ke6Z4gEzeUy>7`(9fPKZzI^zVh5 z^j` z_te)#T(^5ZZ20luEC~T$ovwadgmpv#J#5ci;V0B7e{76=PZ_MaFRj_>_xUC@%t!wl DG${*- diff --git a/tests/queries/0_stateless/test_dozlem/arrow_all_types_2.arrow b/tests/queries/0_stateless/test_dozlem/arrow_all_types_2.arrow deleted file mode 100644 index 04e1214ef847f695b4ed47d72d8f6394f7143569..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1872 zcmds2J#Q015S@!J=aaF(MHHP7imsrbz={n~lmd{2sQ3jGE{GHZmP|n8km%wf1w{%Z zQlxYhgOTqi;uL%NXwr&zCje-tczU+m5$`_9R3|JKR(r5>4_FYRx@Sn;#~ diff --git a/tests/queries/0_stateless/test_dozlem/arrow_all_types_5.arrow b/tests/queries/0_stateless/test_dozlem/arrow_all_types_5.arrow deleted file mode 100644 index 29de25ee984f5d4510ab4eed2692df87dc5f5984..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1816 zcmds1u}&L75S_CR`$VkB1&Umt=t`GPf{;rW6etxX1qDbLummPbE)r!*N{SSet|FI| z6ci~aTw1AIAxbK03X;uxdo$*=;(owm&CR`iZ)a!ccI}*VNmZmNv}I{XO_n6V8!U=} z1t`HY)~Co@pixC+2i}H1MIyKGpYUCH2mS=j1bzuWf_LFDeEiorB(8A1RUJd{x%c<7 zel|Gi6W>9XBF+l>7BF+r7^&y`@;~R8FGgh1VeU8J4hCoa{cqXV!SGOK*mI{uTZ2|* zSc5<8GuKv`w^&1+d5=%`vh_9jgPbB2IWK5a@K*0!`{^S_YuIlMKUyR=`5fPqVt(@9 z2ScWpM`(J$4<*_qa4MCV*GNQt%h#3n`?cIG=QA`Zv>PQ_&Fw2+^BRe$&r0-G_lI?n zU(m2`qY|y=rc^F8H}%tkzIaZkXHUAmhq3P=PiIqBd{68C#5VOj=lf2<+jn5}(-9MB z6RE?!MT}V1wQwI+@-ZmhVGZ7hhR-b3*q?!W<#}gXV0b%tXP$BPSwF(u-p}7&#&ylP zj&o=>kKc-BIQ8fSw-w|1dFyC2eOIyZy@20TY<<}Z`0aq-4frF)wvR6Xe-`jR0{&dF zt$z{lmx?X_DB#Bdf2G)hVe7|rp#N<`jfPD>(Q)hFRI%|h#a2Hd{}+v4)TrBD*ueYG G&HV=)4Vsbw diff --git a/tests/queries/0_stateless/test_dozlem/dicts.arrow b/tests/queries/0_stateless/test_dozlem/dicts.arrow deleted file mode 100644 index c2af6dfedeef8163be1745e3cf5b28be78702d67..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20030554 zcmeF)b-0{qeK+vmopVm)WRlJ9X0urrc5&z#p98_&DJ{jVfWwUy3hN4l0%aBO5eYEvwWWG`evS)d2Hs)BX>T(d*&v0xyzmI)~}jpcy?9o zNUD~rR~u`oHfJnDTdc+0yf4#qw=LP3?5N$DqH0s7-jPQR&s&S#b2m=TPP*y4p_}Ic z$>u|IEFSu|cj$gE*tk_(cnBo`-FORkYzE4fbc zBT4>rHB83IG+9lqpWGn1QF4>yX2~s*TP3$iZkOC4xl{6^$&V#>P41T5Be_>{pX7eY z1CpOgekOTP@{r_V$s>}VO@1!6YBzu#6$)(BV$+ME@ zB+pA;kh~~)N%FGf70IiT*Cek^em!}8^2X%N$y<}RC%>8ecJhwooyohC_az@p_9uUk z97sN%d@A{)COm36hKDkqJm*lR=-IIGI z_e~y<{B-i5fsO)f|-O0Je%Gr3N(k_?kcvYPBk zZj{_KxkYm8zVeDb8^smU)V&rJ3umnF|io}0WNd2#Zx2ZOplfiT_8>|jj zhW+7SI2?|KqtSRY8BIsC(du|*+#e6d!|`Z59#6*8@oc<0 zS()@FgUN6*nv5rt$#gQCtWH;^{pnyjoQ|gB>0~;c&Zeugm05o_mgvjBHfS{~Tg`%2Gtp|cd8OOzY-Kho+mKae4Ow93&*X<*Qum_rs`9e(y7I#E%JS0s ztNp=B_sk%#aM&M=`h#(QFzFAb{lTn1SnUs2x|xP~L-KM*{o%Mjob-p&{&3bGuJ%VO z-SS3*{wVKPUi-K|n)FB0{%F=8t@g(&-Kxfe{&?6Q=k3fJG3k$|{qd|nUhPj-x~-TD z`jcUQGU`wAZs%Q@_9wIcWVJtC={9gW=ue0J>8L*)_ow+|p(hO1JH^L4P*v z&qn>(xIdfpXZc&@{mR?3l9!b?slPhxua5ewf)9`b=0 ztPX}N-CLMvhWTKO2E*}SI2jD5gW+s2Tpf&7x_37-jq*9lr(`@BO$MXsU^E+yRtMvi z?v2m##>2rlAG3U9CWG;GFrE#@tAoi(_czI^Cd0vGG??U5n9t92FqsV|tApuE_vgx1 zOoxN%XfPcQrumTO!!#RAR|m6|?k|}QoDB!F(O@JKtoUU{q-uz9ns_AGr9S^73d9&NihSSyIY^D2L=g*a`n2mu29z>Gp>Fk+bb9qtVKEv@#j3 zOh+rT(Mmpr*)6lL1aF~jaNsLm2Q8{$1v}1W}4)vm1EL$G?|SitE1^kw=3rpnm0bnn~q1*9Mp1v znvJHbquENgm*>Nqze!d#8;@p_(JaTj9IbM2TFC(*pX>a&vK2X2jaDb4)#+$8hs5lk zd7X#bqiZ=e9BudQ`oG(K3yW!e=G;E-E+D&MIropY^Y-MCuAO&q-aU7C$R*MLbJ%}m zIfvVB$IjQy)N+x!ke1w1wxVmTZpr$#W_{agd)je3(vCYN?YNz3$L&fxZg<*oXQmx@ zcG_|0rX6>F+Hn`A9d}XMaaT<{?&@jBT{G>tYo{G|-L&IY(vBOX9XColZjyG~EbX}K zr5(2??YJAJ9e3lj<8GRE+|ARDyJgyOw@y3mwrR)RKJB9rr6~$2~LcxV5z7E=fD?vb5u_NIUM? zX~#V`?YQTs9rwbt<6fM0+)LArdwJS%uS`4c)oI85YT9wXmUi6h(vEvW+Hr46JMJxM z$Gt7>xZg-S?zhs8`<=AoemCv7ccmToo_cRua__GXq%HTM`fyrvzgNGX_S_%VN7ADE zXnibgx=++6)2jP)eJ1U?KdwJX%kIzWvuWFXu0EgE-52XiY2SUNzM2-^!TQUz@&3C0 zCat``tFNV<_l^2yT6*8E@1(8w_w~KB_WrT{Deb*~seet2@89b`(&qcG`tP**@)2sS zZD26=d^R% zdF}jmLA$VBrCro6ZdYwrYgcdAXxD7lYS(VpY1eH((pFl(4cf4c+PF>Hw9VRTyI#9~ z+tY5)ZrEJU<3+)%%FSSRv$F#?`$F;||C$uNFC$%THr?jWGr?scIUv9tBp3$D!_O`XQ zuU*nEZI`vn+ZF9u?b+=)?YZrF?fLBm?S<_{?ZxdS?WOHy?d9zi?Un6S?bYoy?N{4t z+po1>Z?9{wZ*ORCY;S6BZf|LCZEtIDZ@_WAaO_Qm$4_T~1K_SNndS=l|jS zKb-$>_y6twzuo`0b2YBU)wmiz=jZ&KpR1EmC!FIp)V->ERrjjyReihq zcJ=M*+x277k3~Ng{aEx@(_c-0HT~7}OVuw`zf}EF^*`7DT>o?Z&wbB?@0svD6TVNz z_sRG^8Q&-4d!u}Bl<$r5y-~g&&iBLlemLI`=X)T152Wvb^gWQi@74Fc`o34+_v(9{ zeXq0cb@si^zQ5h~xBLEf-{0pZopurxZWu z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`FS;d z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^XQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAJA;;^+LFpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax&nQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqSz`Hi3RbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{QNbIpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSzmCw)X&@j)ima(0O?^dsN)y7(?%^B-;k1Z_L?%W)hxAiHObH7)MnPNGQ z>`8Vc2j}Zg=Vrsv(c{MUA6ZUSx5Q)T>n3iQdm&3-%2sU3I<{mjTeH4xjUQfNAdcC)odOh`e>h;v?sn=7lr(RFJ zo_am?dg}Gm>#5gMucuy5y`FkK^?K^{)a$9&Q?I8^hM)6ue$LPNIX~y;{G6Y!G=9#{ z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^WSXz zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-H+ zEPl?<`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^A8q3=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPN`G*@n=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{L{tH`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAEp3#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=luM>#n1UUKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX}ODQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lq^fV=dL@jP<(578YxFZVt@b`V`B#->bz;v7ATtBs-FW^Yy25v*GCI zabx?BET^hl;<58}6SvI0kfkqWD>h{vTe6m|S>LvDz2SPp^@i&W*Bh=kTyLCs*Bj2~ zduy%s)g^UlT~?RZ73F;HeC~YieC~YieC~X%{$BmP`g`^F>hIOxtG`!&ul`>Bz509g z_v-J}->bh@f3N=jyQThK{k{5o_4n%U)!(a!QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=Z|XqoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-J)Yy6y_^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=T9zv&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS$zie$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaD~g}QdKj-KCoS*Y^e$LPNIX~y;{QRWG&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^etu5zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Y^viLba=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-JKZv32|^K*XA&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=hrEI&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqvJ4(9C94GG9HKQ=X-0d_SGeIX#5gMucuy5y`FkK^?K^{)a$9&Q?I99PraUcJ@tC(_0;RB*Hf>j zUQeA2Kj-KCoS*Y^e$LPNIY0lA#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=luKz#n1UUKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lq{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX}NqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX}Nc@pFF8&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxKhyX*Kj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;Unzdh&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aazi9lNpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqyl%OFVYIZt_+eY9UKs%2sU3I<{mj zTeCjb8?HB8Z@AuYz2SPp^~Q;Kz2SVmx7KQ3T~e3UWp#NupF5vBpF5vBpF5vBpR2!D zf3N;t{k{5o_4n%U)!(bXSAVbmUj4oLd-eC~@73R{zyD6DzgK^+{$BlkO08OJ8`?r! zY)frp+tfCQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`T5^Be$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaClx>E=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`T748Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzQyM?#=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`FXGSIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LM?FMiI?`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6Zj^X-kF^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=luLxji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{OscA{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX{0z@pFF8&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxuU-6{pYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqF0H5m}gka*v`Xut5>^fV=dL@ zjP<(578YxFZVt@b`V`B#->bz;v7ATtBs-FW^Yy25v*GCIabx?BET^hl;<58}6SvI0 zkfkqWD>h{vTe6m|S>LwWUMFQuC)X);YPR;YZ0G6Ok~8YeI;$Lq9ETi-PR!$w{d{k& z)xNrT=cPs>@ZEt1eevuDV=xx$1J& z<*Lh7m#Z#UU9P%Zb-C(t)#a+oRVTyG`8hx5=lqQdKj-KCoS*Y^e$LO|-uO8`=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{DzI6^K*XA z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luMGji2*# ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{PxAq z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJBG z;^+LFpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIY0k&QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS%QW_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax@7wq}Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHaxA71>NpYwBm&d>QdKj-KCoS*Y^e$LPN`HAW0b!eDpSj*VX!*{D! zyJ}-C)#i-#y2ln4YjWgT0xmaSRew%T4NWlbm7DRpYL_Oxv0>DiJq>dZQ;9ETi-9EVQK z`n7x}+|x%SyXKyF$Cd`P}*3`P}(jU9P%Zb-C(t)#a+oRhO$SS6!~UTy?qX za@FOk%T<@FE>~Udw@Y2Fx?FX+>T=cPs>@X;!_WCSKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LOo+xR&@=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KFUHqJ%^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=TC0@oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-ILi=Xpze$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KC{IcTb{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~al_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxuW0<7pYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqSz%Zs1$bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QR88&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^etxau=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxPgFmzL&H47TE=!B zzFWQ8RU2!mHfOBYJ+`n|yK{43-qxpB&i!63W{TxJvM1S*9GtH|otq6uM~@rZe`GmT z-4c(Tuba4K?u9IUDO<5A>)4XDxZZHR;d;aMhU*R28?HA_)awoB^S!lJ`|6Uqw4BeK z&z;Ym&z;Ym&z;ZJ-@m2Q->bh@f3N;t{k{5o_4n%U)!(bXSAVbmUj4oLd-eC~@73R{ zzgK^+{$BmP`g`?o{G6ZjbAHax`8hx5=luLSji2*#e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KC{B4b&^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luNo`8CKJ)D7!Kb>q58-L!62H?LdN zE$dcw>$*+dwr*EkjjM4ruEy248du|LT#c)7HLk|hxEfdEYFv%0aW$^S)wmj0<7!-u zt8q21_P)|DRlijIQu#SQ=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS%Q7@pFF8&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`5hWR=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`Mrvt^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=luLj#n1UUKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJAn#?SdVKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0ka@pFF8&-pn&=jZ&KpYwBm&d>Qd zKR;3ZybcZX3~L$NdH8PiYFBNnrP`daUia9-V(re&fq7e>VmbGFwU{ZE^T?iLM{;n! z{&a3O934GwZ2ytvRCP-{cD`=nmbn+Q^rdXYrmSO2*5Z1@^@i&W*Bh=kTyMDEI8m=R zoX_{xTJ5V#>e6yPcRqJMcRqJMcRqJMSAVbmUj4oLd-eC~@73R{zgK^+{$BmP`g`^F z>hIOxtG`!&um1iKP5r(4d-eC~@73R{hvVn`oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*Y^e$LPNIX~y;{QL>U&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*UD!&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqPA$^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=a&{g=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aamlZ$f=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5z6pYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QR25&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqJi}VXb{@W4z1me9YpFJ8 ztk*rZuvoiub70=qr&!MYUM*&dfkDafZxMl8z zEPW|ku_^1=lC^Bj`nJ{fIw@;9xlXB5v$dyXJ5SG+oKbwA@AG}W&-eNMwHx2(`+T48 z^L@V0_xV2G=lgu0@AG}W&-eL0-{<>$pYQX1uEzKI{*~AF`T5>jt9^Az+0X6g_H%VI z>SWZ(sFP7AqfSPh%v+l}8Fe!1WYo#1lTjz5PDY)KIvI5`>SWZ(sFP7AqfSPhj5-;0 zGU{a1$*7az=lqQdKj-KCoS*Y^e$LO|Tl}1#^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=X)AI=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`E84z^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luNR#n1UUKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJBG#?SdVKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0km@pFF8&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS%QW@pFF8 z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq)4XDY|Z+%)%H3mYdX13sZ+DHr)4`&&z77~t~Xq7xZZHR;d;aMhU<<0f3G*3 z&-d0^?W;@5{}cLuLjOvq@euG?L=yKZ;guAhy5Hu~AQd zKj-KCoS*Y^e$LPNIX~y;{QTREpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e*T}u&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e!ivnIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LM?DSpn+`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^R11a^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luM##?SdVKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS(n6_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax&uaXfpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;*C>9@&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqD+8MI(ppL{v*q&>XvxyeBHz? zb1!7+OWBG|S%>40h;v?sn=7lcde#gPraUc zJ@tC(_0;RB*Hf>jUQfNAdOh`e>h;v?sn=7lr(RFJo_am?dg}Gm$?$W2&d>QdKj-KC zoS*Y^e*Tum&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=kFQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6Zj^Y=A=&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`NxW%^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=luM3ji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{1c6z^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=luL0#n1UUKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX}NwQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX{0$@pFF8&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq=jOn?txvI>`@LGs6w7&JPqHI9 zIA4D{Hye(Q9yhlC$a1Q>B_2CpH*w3{3t9S7wqjG(;d;aMhU*R28?HB8Z@At#k*_zL z&-d0^?JMVV=X2+C=X2+C=X2+C_4n%U)!(bXSAVbmUj4oLd-eC~@73R{zgK^+{$BmP z`g`^F>hB-c)ZeSWSAVbmUj4m#IDXF0`8hx5=lqQd zKj-KCoS*Y^e$LOo)%ZC-=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KFQT&{r^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=TB(-oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-J0i=Xpze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCd|&Z%e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS!c@e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaOB+Au=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T6eR=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv-N%3=j&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LNd*7!L;=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-IfEPl?<`8hx5=lqQdKj-KCoS*Y^etzZl^Ex!lGpuE7=i$56t6jCRmTGgxdfj6S zi?us92j*>kisjtz)ncYt&Lex09m&D@`qR1DaCG#zvHeGuQ`If;*!jANTjpNK(wDLo zo3f5ASy$b*TYFlz^Ym=V8FglzRcF^Zb#9$k=hp>wVO^yzs*CHY zb+zJt-0zx=`*A<+$Njh;_v3!tkNa^y?#KPOANS*a+>iTlKkmoQdKj-KCd{+FNpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa>otDP&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaTNgj)=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T0kSpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QTpM zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze*XF5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hwoXXEGmoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmTIm=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hv-aPf0~&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqVmbGFwU{ZE^T?iLM{;n!{&a3O934Gw zZ2ytvRCP-{cD`=nmbn+Q^rdXYrmSO2*0MG0+g97_q^#-WI;Bp{)}EH_JUv@-Mx9w_ z)!B7Uom=PC`E@~ESXZfw>f*X;U9DVixZZHR;d;aMhU*R28$a0V4d?T{wN^ZU2k-#b z?L2@7@Br;|?Q`vO?Q`vO?Q`vO?Q{L<^rzFGPJcT6>GY@5pH6=|{ps|lduY?2PJcT6 z>GY@5pH4j-Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; zk1c-A&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*aa$2We?&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=UW;-=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KC{Pg1I{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX{1K@pFF8&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHaxuU7n=pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqh;v?sn=7lr(W+GO}(CaJ@tC(_0;RB*Hf>jUQfNAdOh`e>h;v?sn=7l zr(RFJo_am?dg}Gm>#39B=lqQdKj-KCoS*Y^e$LPNIX~y;{CrycoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-J~Dt^w-`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^VP=B`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJAw#?SdVKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX^#8{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;w`u&GpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;cPoC* z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz zJsLmf=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`S}+bKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzgNmQ?bAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{QPUh&-pn&=jZ&KpYwBm&d>QdKj-KC{6zQjIyB5PtYvKH;k(tVUA3{6YIDYV z-D3-jwL3Ql=52k7<=pSpVy0NmBYTn^$-(*h)4ADjbo98f{YREl)h+SZ`MQZ)=3dCs zm$DVEH(YPH-f+F)dc*aG>x~ordc*mAZ#kbkpF5vBpF5vBpF5wczgK^+{$BmP`g`^F z>hIOxtG`!&ul`>Bz509g_v-J}->bh@fB%rC{$BmP`g`^F>hIOV@pFF8&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJAf#?SdVKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0lm;^+LFpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~Z6{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;dyAj*bAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{Csoc=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hw|*Z4U<=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCe0SsL z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX{0% zQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS(nG_&Go4=lqQdKj-KCoS*Y^e$LPN`IX<#>(DUI zu$Hl%hwoOecGbpOs?8beb&oA9*6!RKn78#QmUF*XiUC(b40^ z_8(bJRky@r=j$eJnR_8iU&>Z&$~v}WEnBm`ZMD5l%9>8DQ|i=g?P=N0)3YUK)R}cw zon7bDxpiKhUl-Jcb(OlPE-voJ{jT1)ANS*a+>iTlKkmoSWZ(sFP7AqfSPhj5-;0GU{a1$?$W2&d>QdKj-KCoS*Y^e$LPN`K0(c zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; zv&PT)IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze*W&p&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmSPabAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZPy76;<&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZvMDcTe&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZPz43E?&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6XZu=qJY=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqVmbGFwU{ZE^T?iL zM{;n!{&a3O934GwZ2ytvRCP-{cD`=nmbn+Q^rdXYrmSO2*0MG0+g97_q^#-WI;Bp{ z)}EH_JUv@-Mx9w_)!B7Uom=PC`E@~ESXZfw>f&;};d;aMhU*R28?HB8Z~Wk|H=NIT z01w~+uG@J458wgX=i2Am=i2Am=i2Am=i2A`)9FvAKb`(``qSx8r$3$kbo$fjPxs)a zKb`(``qSx8r#~G(=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=luNZji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KC{L#hF`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAJA>#n1UUKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAG;5{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;&n$k<&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-H!EPl?<`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6Zj^V1tY=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`NhT0`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJB1;^+LFpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=RXX7UWbNxhP90CJbbr$wW~JPQfyiCj*cETw*Sa-s=6f}J6|_(%iIfDy5o@JkmJw~!*OV1 zZOXA^OO7U6b6nY0+v}tpQBJN?>eL*S)bXg}QOBc>M;(tko~t!=JnDGV@u=fb$D@u% z9gjL5bv){L)bXg}QOBc>M;(tk9(6qGc+~Nz<59=M&-pn&=jZ&KpYwBm&d>QdKYv5x z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8huy z7eD9c{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN z`R^7#=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS)yK_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax|FHNuKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax4>W$x&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqSzXN#ZnbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{QPc>pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QPH%pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqyiCj*cETw*Sa-s=6f}J6|_(%iIfDx^{(jg?5E@g?5E@g?5E@g?5E@g?7b>(5~26 zo6QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJA{#?SdVKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ9W#n1UUKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAG;9{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;&nSM*&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKi^sWoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-H!D1Oe*`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^IeUf^K*XA z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luLdji2*# ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{G#IL z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0mP z;^+LFpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=RX{NUWbNxhP90C zJbbr$wW~JPQfyiCj*cETw*Sa- zs=6f}J6|_(%iIfD`ck%HQ`WI1YuTFhZL95dQr2{Gol>V}YfsB|o}Mi^qt2|e>g+nF z&aLz6{JNkntg96F<9=6d+>iTlKkmoiJuPf?V_3V01J-41$&oBLE^_$gimY?%;e*XH#&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAGO0DnIAv{G6ZjbAHaxN5#+i zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*TW) z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hwI zG=9#{`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^LI9W&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPN`5zQN=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPN`9~T*|3B>9XK*CldKd8j(ab1}T48s!TJ0NeurbcLaF8=pIcF$v&gGnO zfpgYK+VH;by-?tspujmnfwSdMa?ViBmIIV?gPfK$pRmq=kZTNLnSD)@V9X{+# zpQGRZS$@vX`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6Y^clbF!=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KFtnqVx&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKmWb(bAHax`8hx5=lqQdKj-KC zoS*aaSB{_Obm?oERcq^PH(j=~m`Rqa)eXl+++|-K;#Ex>SpC$`8hx5=lqQdKj-KCoS*aa|K0dGKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;|0n#MpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHaxN8#uEoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmWq;bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Xp8b9af{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`4=~S&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaf7|#uKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*aauM0ot=lqQdKj-KCoS*Y^e$LPNIX{0~{5+>iU(2joTW7oJvYo|DvRtihI5rw* zr#8oZoA;R3ewT6enAVZqS~qJw>E|EUX5pfFrFC?Eon*M;<@1KltKEl&u1AhXjz^A1jz^A1j^|aI9FH829FH829FH829FH82 z9FH829FH829FH829FH829FH829FH8291lO|=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzcZQ$y zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QQj@ zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz ze{KAnpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*aa9}Yj~=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`T0L-{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;Kic>?Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;zZibb&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax|3~BJ{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0l<@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T4`* z=Q&;aT4vSSI@?W`?JQ=J@Tw_R%&*zTJwap ztY@QU2-k8w-sj%u-sj%u-sj%u-sj%u>TJ~6sIyULqs~U1jXE23HtKBD*{HKoXQR$W zoy~_gbvEj3)Y+)BQD-9$Cl4nNClANZ`8hx5=lqQd zKj-KCoS*aa|JC?8Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;e-(bt&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax7sAi^IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*OjF=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8huyH-65~`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6Zj^Dk=roS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Jy!_WCSKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS$zse$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaf7AFmKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*aauMI!v=lqQdKj-KCoS*Y^e$LPNIX{1V{5+>iU(2joTW7oJvYo|DvRtihI5rw*r#8oZ zoA;R3ewT6enAVZqS~qJw>E|EUX5pfFrFC?Eon*M;<@1Klt$pYQX1zW*wX@AG}W&-eL0-{<>$pYQX1zR&mhKHum2e4p?0eZJ54 z`94?U`+WcL@qK>2ms$36ki*=`-Q3IlJeTM5Lijm9=jZ&KpY!wo(D*q&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax)t~coe$LPNIX~y;{QM2V&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKYydf&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5(1 zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze*TXeKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzk2HSH&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSzFNB}-bAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{QSKeKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzFExJ7&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqSz?}VT8bAHax`8hx5=lqSz zgX8BpUHV#P)!I7SO_%K~W|HMq$TVxHbzH z%`2^=^XnwT6)&GRbZ&L_sIES#r&y|ctkhjr>warl&qm$pTFfiVE6gj*E6gj*E6gj* zE6gj*E6gh%oOy+D`(9?*&p{4zCwFr%_w!ty&kHebH*Pm>H*Pm>H*Pm>H*Pm>H*Pm> zS5K#&PCcD^I`wqw>D1Gyr&CX-p6gm+esi%{NQ%|R!PCcF6D?jJw{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luMW!_WCSKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0l@#?SdVKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*XF4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8huyHGa;|`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Dk`toS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-Jy!q53RKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aamp6XS z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqay)W8 zay)W8ay)W8ay)W8ay)W8ay)W8ay)W8ay)W8ay)W8ay)W8{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;uOEKS&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqHKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz8#aE<&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;Zxepb&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Y^UE}BcoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmXyz&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKmYmgbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Y^SL5gWoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKmWzX&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmYCUbAHax`8hx5=lqTJ~6sIyULqs~U1jXE23HtKBD z*{HKoXY-*=osBvhbvEj3)Y-_x$-~LR$;0t;e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`S~Y>pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e*P(qpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`T6ICpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hw|3_s`R{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`SaoD{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX}PN_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxU)K0JKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*aauMR)w=lqQdKj-KCoS*Y^e$LPNIX{2A{5+>iU(2joTW7oJvYo|DvRtihI5rw* zr#8oZoA;R3ewT6enAVZqS~qJw>E|EUX5pfFrFC?Eon*M;<@1KltiTlKkmo< zxF7f9e%z1yaX;?I{kR|Z<9^(a`*A<+$Ne5J_v31NnPoo*In15h&Ar^ub5S>|ZdTna zKj-KC{A(LO=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`MJ7Oe$LPN zIX~y;{G6Y^Uidjb=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-If(D*q&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Y^b@(|y=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-If+xR&@=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Y^d-yp&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-J~+4wm>=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEQ204N=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LNdxqhD0rLSdHt*x`&blJ{gCRwglHyj&{vs0Vn zzRi0~Yro65dQ9ucZmpZOp7isNYqN0CywW;4zfLk-@$z{?=T>Kr>gtnvilw^8O5J6( z?zfipY}B2u<$7+^)84G-yj4$fJ5S`vJe8;OOrDK-g?WW}g?WW}g?WW}g?WW}g?WW} z#VdDSVcfo#S@v^~!`#W;+{^tu7w*UXxS#pC`MLSI`MLSI`MLSI`MLSI`MLSI`MJ7T zb+hVb)y=A#RX6(~P2H@zS#`7OX4TEAn^iZfZdTo_+$%rl=lqQdKj-KCoS*Y^e*Qa+pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*Wj-=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTQdKj-KCoS*Y^e*Tw@pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e%=~C=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^NsLxe$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS#1ze$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*aaYmJ}tbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QQN+&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqE`2StYHgkErptB~Gs$wb zy5ZPpoSoVn_if%|TKiqb)ni&mc5B_N^`xJFT$_c9=9Sjb`E`=vikHtDI=4D|R9BzW zQ!LdzR_ZRRb-%T&hwt-!zR&mhKHq$pYQX1zR&mhKHum2 ze4p?0eZJ54`99x&+QdKj-KCoS*Y^_2>MYpYwBm&d>QdKY!iubAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEYxp@o=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEd*kQ) zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKYy$6 zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zv zK=?U7=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6X3HGa;|`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6Zj^LGnB=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{5={!=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6Zj^A8L^=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lq@*1B2iNk9L%HVYTc zE3KpR>mpg#)YGY_Q%|R!PVSYT^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIY0mH#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIY0li@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0l4#?SdVKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX_S1=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTyvEP@IX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e!d=l&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaXTs0QdKj-KCoS*Y^e*Tr==lqQdKj-KCoS*Y^e$LPNIX~y;kDs6Cbm?oERcq^PH(j=~m`Rqa z)eXl+$ zpYQX1zR&mhKHq=*e4n51WtROMQdKUaUw&-pn&=jZ&KpY!wA2|wrO{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPN`L~3h^K*XA&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luNj8b9af{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`L{KG&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`S*vP z^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luMw z8$ajg{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN z`42XJ&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPN`Ok!(^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=luNL8$ajg{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPN`Oh_e&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPN`EQ1w^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN`Ge=@ zIbHf%X4Tp{+fA44EM}7BYIVc0(KtJ`IquuM$F%mljH}1Aj_lUDS?ftZ|F|{_7tJfJ zqx0(|!xb-|H*{`w_NcBtsi#<~d#u!5R_lIiF|RPMFt0GLFt0GLFt0GLFt0GLFt2#< z<`u^6dzocF2RY20+|9ihw;Q(`w;Q(`w;Q(`w;Q(`w;Q(`x2va9Pp6(vJ)L?w^>pg# z)YGY_Q&0CnO+B4@I`wqw>D1Gyr&CX-o=!cT+$%rl=lqQdKj-KCoS*Y^ety#UIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN7KKz`Y^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=YJZ0&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=YQV#IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN7JN%rV^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=Y9A&Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QO?{IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LM~8b9af{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`E!k*^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luNX@N<67&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq@*1B2iN&O>w#j@~<#^19&I=@abT=DXGL-&@2jOyx>dWxmG z$4cF0weH9F`99z0`+T48Kil{|-{<>$pYQX1zR&mhKHum2e4p?0eZJ54`99z0`+T48 z^L@V0_a8^!=jVHwWj_Zw%$?i~Kj-KCoS*Y^e*RUBpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lop#IX~y;{G6ZjbAHaxUpxGqpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa*KPcqpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax-y-~+pYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaw`%;HpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax-zEH< zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa zcWeBdpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax-#`4EpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqe$LPNIX~y;{G6ZjbAHax`8hx5=lqufh&wzHT?maEkb$42Aq)aJNv^B&XM?=r3)(>k(S>t?MdHTiqRvha$=-?KeBzfLk- z@$z{?_m+i>>gtnvilw^8O5J6(?q^D1Gyr&CX-o=!cTdOG!V>ghhP zsi#vQdKj-Hk7kQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^FIkc=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^FM3+oS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-J46@Jdo`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Zyrq&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=UwQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-I9g`e|t ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{L917 z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QS}M^PDby zEwgHEo$aQ}b`~?qa<#hQ*l3)c+8p<7-eX$(UB=a8T1R$k-K_PbpMPANg^T8u*3tQO zlHrP%&l@_oI(t-CpVU(<)jd|~E~|CFwXA2O?sP5JbEBU2W$pYQX1zR&mhKHum2e4p?0eZJ54`99z0`+T48^L@V0_xV2G=W2YP??0Np&(HTV z%YF`Wm^<-3_dWMLmy?l`k(2q#CMP2&BPSy#BPSy#BPSy#BPSy#BPSy#BPSy#BPSy# zBPSy#BPSy#BPSy#BPSy#BPYYp`8hx5=lqSzH-(?`bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QPwqKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqSzw={mv&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEZ}>Ss=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq4~bAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEf8*!;oS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKY!=&bAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZvRQNeR z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqE2 zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zv zOylSLoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKYzdQbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZvdiXg%=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6XZn0}tqrLSdHt*x`& zblJ{gCRwglHyj&{vs0VnzRi0~Yro65dQ9ucZmpZOp7isNYqN0CywW;4zfLk-@$z{? z=T>Kr>gtnvilw^8O5J6(?zfipY}B2u<$7+^)84G-yj4$fJ5S`v7;hMF7;hMF7;hMF z7;ik7;|=fgz09(ogB<2g{GHI>3H_bWxZSwjxZSwjxZSwjxZSv2osBvhbvEj3)Y+)B zQD>vhMxBj1n-6H}Y}DDPvr%WG&PJV$IvaI1>TKlU_&Go4=lqQdKj-KCoS*aa-)#JxpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*aaKMp_V=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`T55;e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaKW+S+pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*aazY9O-=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T1uze$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*aa)$ntE&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzmxZ76bAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPN`J?LRIbHf%X4Tp{+fA44 zEM}7BYIVc0(KtJ`IquuM$F%mljH}1Aj_lUDS?ftZ|F|{_7tJfJqx0(|!xb-|H*{`w z_NcBtsi#<~d#u!5{0{jY@;mgX{to$`?`4+#9ON**=f3B@=kj{;dQUfbJ$XHOJ$XHO zJ$XHOJ$XHOJ$XHOJ$XHOJ$XHOJ$XHOJ$XHOJ$XHOJ$XHOJ$XHOJvkYE&d>QdKj-KC z{3{wi=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KC{2Rm1`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAJBXji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{F@s;=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KC{CmRB`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAJ97ji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{QDX|=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KC{3pZD`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ9Vji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{HGf~=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{MW+I`8hx5=lqSzZ!~_+&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSzABCUubAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{QTn@Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzpEQ2X&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqSz--e&_bAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{QR>TKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzH2j>O^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=ZE3v{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~|jKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSz-Nw)PIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*Q%GIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LOoH2j>O^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxA6-As>C)FStJc=pZn|t|F_SD;s~e7u#@VUO zao^@WrnTQ?Ts@|BWVhDMT2K1<$F*6wXkKX@onI#zu6X&pp>wOVM|Jf{J;hSpW2Nr0 zTK8MadN%4#*K$2K>S=G*bKa^axgEaG_n&HfpYQX1zR&mhKHum2e4p?0eZJ54`99z0 z`+T48^L@V0_xV2G=lgu0tMPrl|LFQYKi|tN`#Fg3x$n8}xtxrgjGWAuH#r$O895m_ z895m_895m_895m_895m_895m_895m_895m_895m_895m_895m_895n#&d>QdKj-KC zoS(l&_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHaxzajjbpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHaxzp3$ae$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS(m0_&Go4=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHaxzdQV#pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`T0ABpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e*VslpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T6^VpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqTJ~6sIyULqs~U1jXE23HtKBNuc@<9XQR$WosBvhbvEj3)Y+)Bk%!~w z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqia{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;AKUmjKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqk2@pFF8&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqiKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqiQ{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;pV|01 zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^VP=B`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAEoO@pFF8&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`T3WGpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq@*1B2iNk9L% zHVYTcE3KpR>m&ffM>&ffM>&ffM>&ffM>&ffM>&ffM>&ffM>&ffM>&ffM>&ffM>&ffM>&ffM z>&eORbAHax`8hx5=U>+NIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LNdJ^Y-X^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=U*Rw&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=ik`)IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LO|H2j>O^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=ie26&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=ik%#IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LO|A^e=5^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=RY2P&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=Reu_IX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPTMff>C=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqV_HXcYu&8%q@RCWn}v(!mDbVub&}zVm(LqIw>o=NSD(~VEY&@XH;gxoH;gxo zH;gxoHy-BkhWGhiX4#MTx%avEx%avEx%avEx%|ESz5KoWz5KoWz5KoWz5KoWz5KoW zz5KoW{rfigd-;3$d-;3$d-;3$d-;3$dwDp1&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=luNF8b9af{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPN`A3DH^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=luK+!q53RKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=luMS8b9af{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`M(Z7=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KC{4*Lq=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax_ruTmIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^em-sdoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-I%ji2*#e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCd?);zpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LMyZ9mWH($_Mp*4Ei>x@>1LlPp)O z8;*^}*{RKO-{w80wcll2J*IVJx7N*CPx|@CwOP1mUTGbjUnd!^c=^1cbE~sQb@fR- z#Zuj4rS7s?_gl+)HtJ5-ay>WdX>ZnZ-l`{2AJ6ysKHum2e4p?0eZJ54`99z0`+T48 z^L@V0_xV2G=lgu0@AG}W&-eL0SL6G9|Izk+e!iDke9wK)eb41&{682^G&vbL895m_ z895m_895m_895m_895m_895m_895m_895m_895m_895m_895m_895m_8Gg>s`8hx5 z=luLj8$ajg{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPN`9BOl=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPN`D-+O&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=Wh~z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Zj^EYe!oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-J)-S{~_=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-H^7Jkmp`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^LK3goS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-H^(fBz(=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`TI0} z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=N}n<&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPN`A0W?&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=l?4FoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KFH~gHR^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=bzsAIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LOA!q53RKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`T0uY=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hv7X#AX?^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=eNSo`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJBs!q55n!|mreUHV#P)!I7SO_%K~W|HMwOVM|Jf{J%#ax z@rLn+@rLn+@y6|Dyzy|4H@wgH;(hLY?tSik?tSik{w2-(T>f7EUjAPGUjAPGUjAPG zUjAPGUjAPGUjAPGUjAPGUjAPGUjAPGUjAPGUjAMlj-T^$e$LPNIX~y;uNHpJ&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaS8x2B zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z|55lkKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqi__&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax-?Z^_e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS%PJQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=l`F* zy9};m+wKGY-#Q@as0A~l(bZ*o&q+CEN-B{V6Eg?Q7%*eV3_2RR@SY)+n7Lxcq!N=% z3YalqkU3y>;6swmYUx}_ReDwS7vGwh`|GN%-rZ+6e3+xN_xi2X!_WCSKj-KCoS*aa zw+%n%=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`T0@!IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPTYUAhpoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmWIlpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e*O#L=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hxT8;zgybAHax`8hx5=lq2A90WSU7< zs?`n0M&s<(=BRIr9<$o-GOix8I&e%XuP0wmzMgzN`Fis8QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`T2*3pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e*SkFKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSze{KAnpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa{~dnL&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxr{U-PoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKi>~O=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTg79;G&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq2A90WSU7pA^D65fL$kLOvhMxBj18+A7PoS*Y^e$LPNIX~y;{QPUe&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5s1 ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa zHwr)J=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`T2K+pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax-zNN=pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaw`=^IpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqX@pFF8&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;f4%W@e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*-P#?SdVKj-KCoS*Y^ex8%1uVr4X?bF?K*~v7M ztW>KTj*Z6It<6#27CmOQ-(_4qW_4t*)~#BP`^Cq#SvqfCY8{?kCmF7I@w}mPtFuRS z^+`R&a@}LK?y^?*ThB%|>rPj4HP`BCuh(-vP*3t;9?HXcB#-8?Jf0`=WS+{?d0Ac_ z9>4>501w~+Jb(xA03N^tcmNOJ0X%>Q@PH2s58wejfCumZ9>4>{+tt&lr&CYI)zs6e zr&CYI&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LN-EBu_F^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=YPBLbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zvm&VWeIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e*Q_}=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTU*YHcoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKmYrUpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Un=G=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT z{P1&r&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LOou<>(#&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq2A90WSU7QdKj-KCoS*Y^e$LPNIX~y;{G6Y^VfZ;e=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-IvvGH?$&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmXzI zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Y^ zZR6+soS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKR;^xoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-J~8Gg>s`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6Zj^PdYp=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Zj^IvHEoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Hk8h*~t`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6Zj^WO|V=jVU2pXX%hYnfMT`*b&5b~4Q*E7j_TW213)Yjf1MMUPqS zcNtfYSsmG{b*t9ne(`Z_md=}(T8C%XNro$4Ja6dS>g-Wno%?Y=?#KPOANS*a+>iTl zKkmoecu;Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KFGyI&N^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=bzm8IX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LOw;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS#n{Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSze&gr-oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKi>{N=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCe5dhqe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS%O|QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;uh#fEKj-KCoS*Y^e$LPN`HSJ_Ia&Hz z=GEFh-A$LBOf$(!wYuThXq?^J9QAF{V^;fJ#?@n1NA_yns`a>Ed|aEQ^X8@2;n{VP z;ffc}8#=c-dsJ7S)Ke_iJyz>3YjwZ%Y-F?UbR}1Ft)BLJJ?8`UBoF4HJe)`JXdcVs zc_L5dsXQI?3iAr{imz$r73LM@73LM@73LM@73LK$#=OF~eLwRYSoo=%7;@ot8P}^th`r#&d>QdKj-KCoS*aa?+8EV=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`S}|)e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*aa?`r&QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa9|}L`=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5&4e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaA8GuYpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aapAA3f z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T2V_ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa zpKtt}pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*aa-v~eF=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`T2)6e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*aa-)j7vpYwBm&d>QdKYwZXc}|wTmU*?dPj}O0C(}%_Qmt+{HX3KQHb;G1 z^qAFtmvQx&)sekgw`x7^7a!MV>AZQVb$E82WVqtR^M=l?&K}j(C-oG|b&u7$%Ua!U zJsa7qJ6*}uT&t(OUeEbJJ;{T4C=ch6JetSyc%I0Uc`8qb`*A<+$Njh;_v3!tkNa^y z?#KPOANS*a+>iTlKkmoQdKj-KCoS*Y^e$LPTNBB8E=jZ&K zpYwBdsr;Ou^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{QQ#|Kj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqR||1b5k z!_WCSKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHaxuZ5rUbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{QQ;S=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hwo-uO8`=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6XZ9e&Qw`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6Zj^REm)=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPN`Byi7&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPN`L~Ck^K*XA&-pn&e{uXgCre+;yjt6*yXmr%X(m~z zRyQ0Qjk8;uqrNSA%xb^OxO&X$$X=~mwI272k887Z-n`U0JiAUZT=C+0L+4g!ixY|y ziW7QdKj-KCoS*Y^e$LPNIX~y;KNx<_ z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; zKiv2^Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;KNEh=&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;KiBvQdKj-KC zoS*Y^e$LPNIX~y;zaDQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;zuEXXKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;e;j_!&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=YLv1&&kr)GOyP5>2A90WSU7iTlzdx<}O|x8fw_19)R*tuxjck_SUCGsmw~M!n zw~M!nw~M!nw~M!nw~M!nw~M!nw~M!nw~M!nw~M!nw~M!nw~M!nw~M!XPo?)%eq!^U zO7&{$)zqu0SL5gWoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{CwQ_IX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LN7r}1-s&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqSz%NjrD=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`T18he$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa*9$-A=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T4hnpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqf6OzxcQ|OXtl?t;4hHB*PUio;P%E zb@r&PKB=czu6wN3UDoP;>)FU=-RVli55y0|55y0|55y0|55y0|55y0|55A*`ABZ1_ zABZ2kSn&h%^Zm?okQ=#~Te+P(c_z>1xjY~9bMtfabMtfabMtfabMtfabMtfa^S6lk zx%s*Ix%s*Jc=hq>f`0Ts*hJ6uRdOWJ3r^={G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX{25@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0lP@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq5I{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0ln@N<67&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqEd|aEQ^X8@2;n{VP;ffc}8#fh$X4K89n^iZ<&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lp!s_&Go4=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHaxKfCdBe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*yu&z}!J=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqsMB`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJA0_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxzdZb$pYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aauWbCBpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaZw)`^=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T6TNe$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*aaZ*TmZpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{BH8|oGg7U^J;CM z?xxF5rkP}=THSDLG|p~qj{3IfF{}M9++|-K@GQdKj-KCoS*aaA8h=bpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aap9(+c=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T4sye$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aapK1J@ zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa zUkyL!=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`S}Moe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*aaUvK=JpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*aaKMX(T=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`T55;e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaKW_Y-pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lq#fPiWkotI`e(L&-eL0-{<>$pYQX1zR&mhKHum2e4p?0 zeZJ54`99z0`+T48^L@V0_xV2G=lgvBqVG?$Tt>fIO21ZKzn+b3meg}KuEy248du|L zT#c)7HLk|hxEfdEYFv%0aW$^S)wmj0<7!-ut8q21#?`pmXGC49x>R+k{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LOA8b9af{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPN`DZnL&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hwo+W0v? z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LOI z2tViN{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN z`Im*C^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=luMs#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=luM2!q53RKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX{2B#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=XaN%=Va+?nOAH3bT?ggGR-6_)#`?0qj7d?bJVv*k6G<^8CQ>4 z9oegOtJdRw@o{aI&YPE7hiBJGhAUn?Z|L0W>``5PQctm5_gJmFtkwP2vysiZ)0JG! zwR+m?^_&mXlRTJ*@^Bu>qj@Zk$9Thd!+67Z!+67Z!+7KF8gKYL-_JY;xsjW>mD{s3B?J;360y0+l||e+l||e+l||e+utneY}DDPvr%WG&PJV$IvaI1>TJ~6sIyUL zqs~U1jXE23HtKBD*{HLT5692>IX~y;{G6ZjbAHax`8hx5=ieWG&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=ReT+IX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LO|CH$P9^K*XA z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=RX;K&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=Re)} zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN7 zApD%4^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=f~ma{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIY0lP#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=luL*!q53RKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIY0ln#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqg-WneNs=cT=!V5 zyR6my*0Yh#y3>_h&9!>k>-C%u)RR1zhw^Y9$)kBJkB9H`eZJ54`99z0`+T48^L@V0 z_xV2G=lgu0@AG}W&-eL0-{<>$|I@?w`99z0YJ8vX-(SAZ&-XLWL2l$`Zsm6FQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`T1uye$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^|Npu7w_j=eoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-JqH-65~`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6Zj^T)%_`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJA%;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS%PrQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lq#de$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS%Pl_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxU$^mde$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS%PdQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS%PR_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZPZGN7UrLSdPt?kp@blJ%? zldM#$8;*^}*{#h{-xfV)wcll2J!W-euhy+vkNd^PwOKlEUTPhlT_+i?c=5cUbE~sQ zb@fR-#d6(aweGT3_gl|K%qz?*%qz?*%qz?*%qz?*%qz?*%q#A;d4+NNe&#vIjoi$w z+|Hdm6XSN{cH?&AcH?&AcH?&AcH?&AcH?&Sbn5Beys4*CPp6(vJ)L?w^>pg#)YGY_ zQ%|R!PCcD^I`wqw>D1Gyr<3=}&-pn&=jZ&KpYwBm&d>QdKj-KCoSz>ye$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*aacMd=2=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T0+TpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T0*Z ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa z_YXhk=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`T4JepYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`T4Ile$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*aaj}AZQ=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`T6gMpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`S~9QdKj-KCoS*aaPYXZi=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`FR>Yf2sL-PL{rwd9}7rchhAj(@e5bt!_9r8fUjQM}1rL znALularKzhk-b{CYCY~3AJ=B-ym_g0cy^s+xZ=h0hR&_d9@W(+^%To>kJY-%THS9w z8{zwWpYQX1zR&mhKHum2e4p?0eZJ54`99z0`+T48^L@V0_xV2G=lgu0@AG}W&-Xv0 zz0`c4pYLa$gWSl?+{*3T$ur^S{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LO;pY!vr#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX{0a{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;UlM-K&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqSzmoQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzH-(?`bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QPwqKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqSzw={mv&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz_lBSIbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{QS)tKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqSz_cwmd&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5=F=Q&yWTISW-KHW{1 zolG;yO0~M-*l3*H+8p(5(PLKoUB=a8R!8<~-KzDtUwmAfrSs;c*5TQ8lHrOM&l@_o zI(t-CpVU(<*F9G2E^Bqa^_W+fSD06rSD06rSD06rSD06rSD07aee(+A_WjIrkQ=#~ zTe+P(F>W_*H*Pm>H*Pm>H*Pm>H*Pm>H*Qx?r=CtdoqD==ih4Trbn5BU)2XLZPp6(v zJ)L?w^>pg#)YGY_Q%|R!PTng&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN`HzR6^K*XA z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luL#8b9af z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`A;@} z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN z`7ei`^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=luKw8b9af{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPN`Eldt{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIY0lX@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIY0l@@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKmYW`&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqu2U~PsLwB+H*{}V%BZeBsi#=3d#u)7*6M!i z;ro1_@AG}W&-eL0-{<>$pYQX1zR&mhKHum2e4p?0eZJ54`99z0`+T48^L@V0_g{*> z&(HTW&p~eFW^Uzn?u4K7bAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX_o_&d)a+Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzvyGqgbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{QS}IbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6YEariku=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-IP+W0v?=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-J)7=F&r`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Ve?toS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-J)-1s>^=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-J)6MoLm`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^EYk$oS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-J)*Z4U<=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=Xax@=Va+?nOAH3bT?ggGR-6_ z)#`?0qj7d?bJVv*k6G<^8CQ>49oegOtJdS1{JpR&z0i0S+rzW#B*PUio;P%FS<0xc zKB=czu6wN3UDoP;<`w1@<`w1@<`w1@<`w1@<`w1@<`sA2yu!GBKl2>qMsDU-ZpXOY zxZSwjxZSwjxZSwjxZSwjxZSv2J)L?w^>pg#-ZARw)YGY_Q%|R!PCcD^I`wqw>D1Gy zr&CX-o~}|VTWd>g)W&Vnrfs>cwAD6iYprYRZKH8DuEy248du|LT#c)7HLk|hxEfdE zYFzEd!qvDMSL142jjM4ruEy248du|LT+Mqby-!A6DnIAv{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Zj^Pgz^oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-J~8-C8u`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6Zj^Ir--=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Zj^IvKFoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Hk8Gg>s`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6Zj^WP0W=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^WSg$oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-J45`NCl`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Zyrq&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=Uw1&x+Yx{IJ zU3N0fBrDbGhGU~~c58Fgw?&Ux?RObhk69hrt97f^<9b=i3(L|AjaRWfJiAUZT=C+0 zL-&@YjOyx>dWz+`$7&e%X zuP0wmzMgzN`Fis8QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSzBjM-#oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmVfebAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6XZ*7!L;=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Y^M))~D=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-JK)%ZC-=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zfg`e|te$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{JX=?`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ9Nji2*#e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KC{CgWe=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lq9nCre+;yjt6*yXmr%X(m~zRyQ0Qjk8;uqrNSA%xb^O zxO&X$$X=~mwI272k887Z-n`U0JiAUZT=C+0L+4g!kLv1^dWz+`$7QdKj-KCoS*Y^e$LPNIX~y;{QR97Kj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqSzk2ikK&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzFNUAQdKj-KCoS*Y^e$LPNIX~y;{QUhIKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqSzFE@V9&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz?}VT8bAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QRRDKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz?=^nT z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa z{~LbJ&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;YvJeooS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKfe`z&d>QdKj-KC{ND8QoGg7U^J;CM?p9{pPNtb;rCQx^Y&6bpZI1f3=rODP zF5~Jkt0Q~0Zq<6+FFvl#(s}bz>+tM4$#BJs=M9}(ojt0nPwFX_>mI9h7vl}%jrC@{ zVZ33yVZ33yac_<{{GRV;o`c-T&GQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTXW{4koS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKYz6GbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEapULw zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmQBi z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT z`tWmp&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hxT#>UV2IX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e*Ra&&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*Y^e*VUdpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`T5(2pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*Y^e$LPNIX~y;{QMmoKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T70o=Q&yW zTISW-KHW{1olG;yO0~M-*l3*H+8p(5(PLKoUB=a8R!8<~-KzDtUwmAfrSs;c*5TQ8 zlHrOM&l@_oI(t-CpVU(<*F9G2F8&VrJLK=s{rY#v_k2I|9OOoB#`oO!-1l6*o_sy| zdh+$;>&e%XuP0wmzMgzN`Fis8QdKj-KCoS*Y^e$LPTR`@wT=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LO|xAAj+&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTd*SE&oS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmYIH=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT$i~n4 zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*U|S zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{QUofpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`S~pToS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-H+!_WCSKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lq|~lrR;twv$42Aq*5;^hiypJu z?=r3)vpTX@>sGDD{o>==ES)zmwGPj&lMGk9c;3*t)!C!E`lOy>x$d!Ack%zlc;AhD zJ>w1I4dV^tjeB*x;rDz$^Bm+x#1F&|#19^d-*dm`e$V}$%fFX@FaKWtz5ILm_ww)M z-^;(3e=q-D{=NKr`SSzBaNT)bAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QQd=Kj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSztB0TS zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QRGX zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`S~|Ae$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*aazZ`zf&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*aay~fY^IX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e*WE!pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{QTdApYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`T37Fe$LPNIX~y;{G6ZjbAHax`8hx5=lqG?T1Us~e7u#@VgSQQsClX0_jCTs>xWWUtn(T95n1$F*5HZ(eF0o?Ryy zu6Xghp>wOVM|Jf{J;id}W3}$GR`*-aMmFnCS8_Gi>S?dnb3RZ{@?iKr-{<>$pYQX1 zzR&mhKHum2e4p?0eZJ54|7Q3;-{<>$pYQX1zR&mhKHum2T#fJZ{rlJV`T2h4ImnIp zp8KBrp39SwCnHZro{T&hc{1{3QdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Y>}| zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Ja z*!Vd==jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPTTlhIY=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZvPUGkNoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmUi}=lqQdKj-KFpS`;bu4G%!1N<&+ zkhIl;nbBx8hw)xJV8%o-bHGeVC1y%0F;n8iZ0*4V_ZkCc445%sTPE2dGbNQ+W;?0G z6mqK4`C8gnQk7i(d*;kMeXFZi_g)P@?$%zbpZA-?&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*aa{~LbJ&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;tKsMToS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKff7%&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS$D0Kj-KCoS*Y^e$LPDT|dw9($_Mt*7nJ6y6j|{Nmi=W4aY{~ z?AGS6Z;KwY+V3*19pg#)YGY_Q%|R!PCcD^I`wqw>D1Gyr;~f-=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hv-?eKGc&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPTWaH=joS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmR9QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e*Qh-=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hxTvyGqgbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QO@ue$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aacMd=2=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T389pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax-#`4EpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqF1;TlARKewT6enAMTJTDNLF>K7l^X6dYX zp>=S2on*M;`SXU(t$pYQX1zR&mhKHum2e4p?0eZJ54 z`99z0`+WbeG``RG`99z0`+T48^Zf_e_xbsL=D87m&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=luLXHh#{}`8hx5=lqQdKmVxkbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZvcKA6z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTuZ^GcbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QOhH&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmR+8pYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=QqO7`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbACQ+{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;w;Dg^=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T1`6IX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LOoIQ*QS^K*XA&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=MOc0&d>QdKj-K7 zvY+R8>1&x+Yx`t3U3N0fBrDbGhGU~~c58Fkw?&Ux?RObhk69hrt97f^qki#mZI;fO z7g`6W*GYyeo(0fAf360ymC-k21?>FOi<96eA<96eAbvEj3)Y+)B zQD>vhMxBj18+A77Y}DDPvr%WG&PJV$IvaI1>TJ~6$iwk-e$LPNIX~y;{QR}T&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmQYr zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzcZZ+zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{QS=}e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*aaf8O{xKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;KN^0{&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;|GM#We$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS(m6_&Go4=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHaxeQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqSz{q5&DUiw<* z)!IJUO_!ZaGs#M|y5ZPpoZZ?S_HEH)R{LGX)nisi_G;a#^{8KbT$`n{=7rY5>2;Ff zis#Q8I=4D|R9BzWQ!Lj#R_iWnb-(p&WV7ycDVKAlp7v@z=e2s0>v<>-=aD>`$MSfd z$dh?0Pv@CDn^)wO;eOnY`*A<+$Njh;_v3!tkNa^y?#KPO-+yS_kNa^y?#KPOANS*a z-nVl895m_895m_895m_895m_8Gg>s`8hx5=lqQdKj-KCoS*aa-wHqH=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`T1XO{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;|E2MBe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS%P6_&Go4=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax|4;ZiKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqjLQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T0EjoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-JGji2*#e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KC{AT0l{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX}M^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*aaFA6{B=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T6z6&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq%TA`5WTjf&aBMWL|N2sL*tbQGS?zZjSC3g8*{gM{)}wy$ac!2) znipCJr`Jh_E1o}Z=-le;QC)pfPvQN6_Xpk|cz@vif%gaAA9#Q8%6Na^{ekxf-XC~> z;QfL32QTgW1M~C!n4g=Uo1dGXo1dGXo1dGXe_hPa&Ckux&Ckux&Ckux)yJ!kS0ArF zUVXg!c=hq>QdKj-KC{IwfD=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^EVAY=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{LLFb z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^LGqC=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax-?{N~e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*+#QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*-E_&Go4=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax-@oy5e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*+<QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqiTlKkmoQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`S~Y@pYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QOfJKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzH2j>O z^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lkL3 z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX_=% z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; zHyS_Z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`T0)xIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LOoF#MdK^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=ev!c^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=luMO8$ajg{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPN`76TD`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAJA{;phCEpYwBm&d*f*D|lx_Q`I#>|~lrR;twv$42Aq*5sGBt{o>==ES)tkv<^x$d!AcUi0Z zt!E>fb*D?YoGbOTSL->i)stM$LwPulD1Gyr&CX-o=!cT zdOG!V>gm+esi#vQdKj-KCoS*Y^e$LPNIX{1s@N<67&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX{1g@N<67 z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX{1&@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIY0mK@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lq&ZKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqqj@Zk=ZQR-r}A{3$+O{p+>iTl zKkmoQdKj-KCoS*Y^e$LPNIX~y;{QQ4~ zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`S~RLoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-J47kQdKj-KCoS*Y^e$LPNIX~y; z{G6Zj^X10R`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbACQ={G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;SHsWwIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e*OjF=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hwo*7!L;=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6XZ8-C8u`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Zj^REd%=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`PVgm&d>QdKY!`{JjY94%e-3KC%fsglW8Vdsa7`} z8;!GDo5Q{>ddzCS%eZ>X>d0QLTeTkbi;ruwbk@AkIyk*fGF z;QfL32i_lef8hOr_Xpk|cz@vif%gY*9PbajKk)wGrGI}g&2qg1Sgp4JYxO>0Jsa7q zHvyM&Ialg!f^oZXyK%d5yK%d5yK%d5yK(z>#JJtK-MC%dth!lsv+8Em&8nMKH>++| z-K@GQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Y3c>oS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-IfAAZiy`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^B)dB=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^TWo^ z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJBb z;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS(mMQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAJ9|;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq5e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS%P0QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAJAb;phCEpHFhK^tH^ZxxLu$vXf~hSxHy_t7SCKZfy?xw&*dd z{VwC`F{>kcwQkjV)Gt1+&C*%(LhIo4I>~Uw^XCnnTb(_stMh%n&-eL0-{<>$pYQX1 zzR&mhKHum2e4p?0eZJ54`99z0`+T48^L@V0_xV2G|DWOee4n55bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0m8#?SdVKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T6IEpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv- zCj6YA^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=U*Lu&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6Zj^RI3EoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-If6n@Un`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Zj^KTD7=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqg-WneNs=cT=!V5yR6my*0Yh#y3?gx&Xs!FtM#1M z>PfEWp*);N@@O8*<9Q-a=BYd#e~0`X@^|Rv@ps7gd_VKt$j#i!?cB-TJeTM5LSD?P z^6HpZm{*ur_&xV~?)Tj9xm>PX?wd5ZT)AAiT)AAiT)AAiT)AAiT)AAiT)AAiT)AAi zT)AAiTz)Q>E0-&mD<{Lx`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^S28>=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{2dxU=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Y;or=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{Cyff=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^A8O_ z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC z{KFeR=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6Zj^G^sr=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqg-WneNs=c zT=!V5yR6my*0Yh#y3?gx&Xs!FtM#1M>PfEWp*);N@@O8*<9Q-a=BYd#e~0`X@^|Rv z@ps7gd_VKt$j#i!?cB-TJeTM5LSD?P^6HpZm{*ur_&xV~?)Tj9xm>PXu3WBMu3WBM zu3WBMu3WBMu3WBMu3WBMu3WBMu3WBM?k6?5Tz)Q>E0-&mD<{Lx`8hx5=lqQdKj-KCoS*Y^e$LPNx%%_Z2|wrO{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`K0l4e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS%PQQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq1V&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmUTp&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-JqHh#{} z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^RH?A zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-If z7=F&r`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^KT13=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX{27 z_<4?(zLt5lwoi7`Whc{2vQn*XI5rw*w>F1;TlARKewT6enAMTJTDNLF>K7l^X6dYX zp>=S2on*M;`SXU(tQdKj-KCoS*Y^e*S~u=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv-`^L}t zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*VLa zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze*V+p=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hv-@5ayhIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e*UwKpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e*UZB=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hxTu*T2%IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*Wu?pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e*OpH=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hxT#KzD0IX~y;4~n1XcV{*Z zadvBS*tbQGS?zZjSC3g8*{gM{)}wy$ac!2)nipCJr`Jh_E1o}Z=-ldTe~0`X@^|P# z`8zbta{Vn?t-mH~^><}G8`-SCD3@|May)W8ay)W8ay)W8ay)W8ay)W8ay)W8ay)W8 zay)W8ay)W8ay)W8ay)W8ay)W8ay)W8ayQdKj-KCoS*Y^em-veoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-J4+xR&@=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QRl#bAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEW%xNi=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-IP-S{~_ z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqY zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-J) z8h*~t`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^EYbzoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-J)-uO8`=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqddzCS%eZ>X>d0QLTeTkbi;ruwbk@Ak zIyk*fGFQd zKj-KC{D&Gp=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{HMat`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAJ9_ji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{AU_J=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC{8z%y`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAJAzji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{MQ;k=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KC{P)Ap`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ8_ji2*#e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCyoI0hbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QR@R&-pn&=jZ&KpWi=zp5vvjWnQiA zlihUL$uyI!RI3|~jmFun&0*gbJ!ZAvWn4XGb!4yBty+)z#mBW-I%{5N9h_b#8LoK# zyrHx4hVjOz8E+VG7;hMF7;oG^BK{6yI~-bKi5{bGcl( zT)AAiT)AAiT)AAiT)AAiT)AAiT)AAiT)AAiT)AAiT)AAiT)AAiT)A91895m_na^o* zGW?vM^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hvV zfBs_lIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LOYG=9#{`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6Zj^H(>1&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX{28@pFF8&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lq{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX{2>@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX{2v@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QPYjKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqG?T1Us~e7u#@VgSVc!-#X0_jCTs>xWWUtn( zT95k0$F*5HYhGv_oL(mxu6X{up|kOZ@rLn+@rLn+@rLon{WRW~X1T@;t2Js^tMS8n zHnLeG2j6qwbKi5{bGcl(T)AAiT)AAiT)AAiT)AAiT)EsIY;w7BxpKL3xpKL3xpKL3 zxpKL3xpKL3xpFdcGIBCQdKj-J~9)8Zx`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Pdbq=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Pg_~ zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Hk z6n@Un`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^Ir}>=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqG=9#{`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6Zj^IvWJoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Hk8-C8u`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Zj^WO_U=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6Zj^FL_(oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-J48Gg>s`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Zj^QFem`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAJBWji2*#e$LPDCqK{e($_Mt*7nJ6y6j|{Nmi=W4aY{~ z?AGS6Z;KwY+V3*19|vBE>|vBE>|vBE>|vB zE>|vBE>|vBE>|vBE>|vBE>|vBE>|vBE>|vBE>|vBE>|v>pT7`(&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{8f#g^K*XA&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luMM@N<67&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;U)lIM zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z-yD9<&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;-`e;&Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;-yeR?&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;KhXF&Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=l7SN=XmLB znOAH3WH()QGR-6_)#`?0qj7d?bJ({49oegOtJb4_@o{aI&YBlm2dCFb zhAW;wZ|L0W>``5PQctm5_gJmFtkwP2vysiZ)1_R_m3rE%^_$pYQYicMIR=`+T48^L@V0_xV2G=lgu0@AG}W&-b|+-{<@H zm+$lQ{mgSCH*+hub0>H6T%M2bx$n8}xtxrgjGT;|jGT;|jGT;|jGT;|jGT;|jGW9* zL{3IdMovafMovafMovafMovafMovafMovafMoxyG^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS(l(QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAJAT;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`S}Moe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaUupcDpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*aa-wi+K=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`T55+e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa-*5b!pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa*7!L;=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6X}hM)6ue$LPNIX~y;{G6ZjbAEnr z`FW0)zLt5lwoi7`Whc{2vQn*XI5rw*w>F1;TlARKewT6enAMTJTDNLF>K7l^X6dYX zp>=S2on*M;`SXU(t{y31PKZ#^3^uXsM@73LM@73LM@73LM@73LM@ z73LN9*1W>FeLwTu$j#i!?cB-TJQw5kOU=05xZSwjxZSwjxZSwjxZSwjxLrM+dOG!V z>gm+esi#vlKt{G6Y^*!Vd==jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{K>}8`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ96ji2*# ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{B^_6 z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ9! z;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAJ9Uji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KC{4K-J`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAJAP;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAEo%_&Go4=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax-!=T4pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lq49oegOtJb4_@o{aI&YBlm2dCFbhAW;wZ|L0W z>``5PQctm5_gJmFtkwP2vk|_}_xV2G=lgu0@AG}W&-eL0-{<>$pYQX1zR&mHz43j% z&-eL0-{<>$pYQX1zR&mh{)6WG{Cq$2+{n$`%I)0A-8>h5&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHaxf3op&e$LPNIX~y;{G6ZjbAHax`MLUYe$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC{FlPd`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAJ9oji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{FfU)=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KC{CC37`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJA@ji2*#e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KC{P!9^=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCJdL08bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QNT;Kj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqSzM))~D=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-K7nxE%*>1&x+Yx`t3U3N0fBrDbGhGU~~c58Fkw?&Ux?RObhk69hrt97f^qki#m zZI;fO7g`6W*GYyeogm+esi#vQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaFK_&u zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa zZwx=@=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`T6TLe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*aaZ*KgYpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*aa?+ria=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`T1Kle$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa?{EB^pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*Y^e$LPNIX~y;|33VjpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHaxf1>eoe$LPNIX~y;{G6ZjbAHaxA3Q(L@zU2auh#a- zZo2Gbnn_lw)eXl+$pYQX1zR&mhKHum2e4p?0eZJ54 z|62Gy-{<>$pYQX1zR&mhKHq=ve4n51XPz6mnOnJ?JGmQv&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hxT&*A6%oS*Y^e$LPNIX~y;{9OGxKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;M~$EJbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QPf)pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QP4YKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqSz?>2tU&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaKKz`Y^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=bzE|IX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LO=!_WCSKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*S3qIX~y;{G6Zj zbAEpB`FW0)zLt5lwoi7`Whc{2vQn*XI5rw*w>F1;TlARKewT6enAMTJTDNLFs>$DL zmZjG;{)p|t>2;Ffis#Q8y0s=xZSwjxZSw@@n+m^+-}@%+-}@%+^(KZJ)L?w z^>pg#)YGY_Q%|R!PCcD^I`wqw>D1Gyr&CX-o=!cTdOEpRe$LPNIX~y;{QS!rKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzpAJ9g z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T0K! zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz z>o$JQ&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;e=hu-pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;Z`t@cKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*-@@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lq?{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0mL#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJAx!q53RKj-KCoS*Y^e$LMyL_g2*($_Mt*7nJ6 zy6j|{Nmi=W4aY{~?AGS6Z;KwY+V3*19QdKj-KC{FfR(=jZ&KpYwBm&d>R|`g4BH&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6Zv&G2)6&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEbmQmzoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKmVP^&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*XW$&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq5ZTBbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{JaZ4=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzBjM-#oS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKmUi}=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv-tnqVx&d>Qd zKff3KJjY94%e-3KC%fsglW8Vdsa7`}8;!GDo5Q{>ddzCS%eZ>X>d0QLTeTkbi;ruw zbk@AkIyk*fGF?T}PjWpE z<>44_7;hMFd}%Y@Fy1iUFy6Qq#~Xgn_cPCp+{~@qj`xJ#6M9c*+-}@%+-}_dr()c0 z+-}@%+^)_>osBvhbvEj3)Y+)BQD>vhMxBj18+A77Y}DDPvr%WG&PJV$IvaU7e$LPN zIX~y;{G6Zj^M4wC&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=ik`)IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPL!q53RKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAJ97ji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{CgWe=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`M(W6=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPT zKYRE6TS>p)1^oT>ImhLA+;+WQZ+V`5>b)r-NN*0w?InO9y@4RTfgrtsAkAmXQ}0a= z2+|t}(i;ez<`h7X-asb3fgquShR@hXwk7`qyVh%@(R^p-JKvdKX6EDbde1ID=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIY0mNji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{NFTw&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`LBha^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luLHH-65~`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Z(HJIX~y;{G6ZjbAHax`8hx5 z=aKZP}}Nv*x3IzPlC+ zXUz+(gVS-6VZ`&t4V_yZJ*v?sbrnl>j+HvgYMpN_>)EI?ZRJX?*419C>%3l9aw9i$ zD}0~t^L@V0_xV2G=lgu0@AG}W&-eL0-{<>$pYQX1zW*uV`+T48^L@V0_xV0oQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hxT&yAn+bAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{QUF7&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*U)`Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSzYWO)n=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lp!F@pFF8&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*ReF=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT`;DLTbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QT|1&-pn&e>nX-$5UU^ ztXkVAtLd_n#Z0nXt!~&h8fUi_hkcv(nAUoiarKzimc5!cYd-4dyKAv<*1XUas;@jiz3V&BToPSrS z+xIie-Q3IlsN2=;>UMRzx?SC_ZdbRf+y7)!x2xNIPv?6&-_!Y?&i8b_r}I6X@9BI` z=X*Ne)A^px_jJCe^F5vK>3mP;dph6K$-VM(e$LPNIX~y;{G6ZjbAJAh8$ajg{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`FqvBTKnGl zGkKr9Z(fyG=QVk4UYFPB4S8eUlsD)7!qvDMSL142jjM4ruEy248du|LT#c)7HLk|h zxEfdEYFv%0aW$^S)wmj0<7z(?-=+F4)px1QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAImsKmY0QbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZvxyH}=IX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e*Ui;Kj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqSzM}(jAbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QOtL&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T4(Z{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;pB#S9&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzUu*oF zpYwBm&d>QdKj-I{)z5Q0^)=0^wSBUhE<0JwB+J$6hHay9c589iw|S3gt#=t$k7;e$ zt9i5Lqkg`-77J(13$26Gagt%g^T!RHTOB>B(I<5kOLdNwI*aF!=aA>nWql60&-XLS z-Q3IlxX<0^?sIuPc|CbOc|CbOc|CbOc|CbOc|CbOc|CbOc|CbOc|CbOc|CbOc|CbO zc|Cc(|J3C5QdKj-KCoS*Y^ ze$LPNIX~y;pBH}4&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*aazt#9TKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hwoAAZiy`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Zj^RDr8e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS#3}_&Go4=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHaxZ-$@qbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QRrK&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmWGybAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZviN??QIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-IT8-+Yx`t1U3RjVNtUbC4ckWJ z?AGG2Z}T40TJJKh9@E;gSMz4gNBw+vEf&t27g`6W<0Qj~=Z_mYw>o-MqfhE8mg*cU zb(YmS-&)qQQD@r9m0Yc>y;j$Gy{_a&Jcm4oJcl0L=aBn+KeOD;z4*Jr-xdC@@ILoG z_dfSNm&=vQmCOA{O)gh1S1wmBS1wmBS1wmBS1wmBS1wmBS1wmBS1wmBS1wmBS1wmB zS5AhX^K*XA&-pn&=jZ&KpYwBm{+{9I{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0lw@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax-yD9<&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzpKbh{pYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqb@pFF8&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;AKCahKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;pVIg_ zKj-KCoS*Y^e$LPNIX~y;{QS}N^BhloO|xokpRA_KP8KuCa<#f)+i0BKS{(Lm-eX$p zUB=a8T3hyN-mLkkpYN{4!dde|>)>>pWEk=MaYN@;M~`asNnOQKonxiWvRda`%X&8I zOk25N>C2mE4Hukmr!+(4+eta-Z*Kmbas!rv9%=icYu=icXXxpKL3 zxpKL3xpKL3xpKL3xpKL3xpKL3xpKL3xpKL3xpKL3xqsN?a^-U6a^-U6WcWEh=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=bsyX&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^UrVmoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-Hw;phCEpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC z{8r=V{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIY0lJ#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIY0l_@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*aaN7>JFJoPorsV|EjadvBQ*tdC)X{~n|SC46J*{gZ8=A(YT zyA}&)%?quA({Ykv#Pi1uom(9}s?jHP6-#vv^@e&wy`kPvZ>Tra8;^3m;eEcJS?f7EUjF{uoBX}}z5KoWz5KoWz5KoWz5KoWz5KoWz5KoWz5KoW zz5KoWz5KoWy*wO0=jZ&KpYwBm&d>QdKj-KC{5`_Y`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ8<;phCEpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqR`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ9qji2*#e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KC{7vEK{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIY0l&@N<67&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0mD#?SdVKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0m9@N<67 z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0l^ z#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIY0ls@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lq2ilh>2ilh>2ilh>2ilh>2ilh>2ilh>2ilh>2ilh>2i zlh>2ilh^x$Ca)*2C$A^3Cnv+t`8hx5=lqQdKj-KC zoS*Y^e$LPNIY0lL@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS%PQQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`T5=ObAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6Xp8$ajg{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`TfSv`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAG-Xe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aauM9uu=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5Pp&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QPag&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKYzQ%&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=a06Z=XmOCnpJE2WHnuOvY1JhtJMwL zM&s<(;;?V?9@ARyGOixe+Ok*kX3aMEA% z94mE})jHo=*0WJ(+RBw&t*gCO*Ll6J$pYQX1zR&mhKHum2e4p?0eZJ54{oc;^`99z0YJ8vXKia;} z&-XKn_qq4E_qq4E_qq4E_qjZrJe)k7JlwlS9!?%k9!?%k9!?%k9!?%k9!?%k9!?%k z9!?%k9!?%k9!?%k9!?&PpYwBm&d>QdKj-KCoS*Y^e$LOoKm44Z^K*XA&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=kMA0IX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LN-u<>(#&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-H^5q{3k`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^EWqs&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=N}q= z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN z`G+@t&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=bsRM&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPN`6o4g&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj^M~8db3FAm&8oG1vYIYC zSKZP}}Nv*x3IzPlC+XUz+(gVS-6VZ`&t4V_yZ zJ*v?sbrnl>j+HvgYMpN_>)EI?ZRJX?*419C>%3l9aw9i$E4TAl9?zXTktg$1p3XCQ zHm}Gl<2mFx2Yx^B`+@hl-w*tL;P(S{ySiQ7u5OpVm%o?4m%o?4 zm%o?4m%o?4m%o?4m%o?4m%o?4|NSO^FMlt8FMls5!_WCSKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*-%@N<67&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX_SzEc~3G^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=PQk$^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luL$QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzjmFRUIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e*RUBpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmX?NbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Y^ZR6+s zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqB(I<5k>J9aVdPBXT-cWC-Hy-tR!~1+c-sj%u-sj%u-sj%u-skf7^7r!h z-`eEwQdKj-KCoS*aa?+ZWY=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`T2V^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaA87oXpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqSzVfZ;e=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN-vhj0%&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LN-G5nmL^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=O5PiIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN-x$$#;&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN-C;XhB^K*XA&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=bza4IX~y;{G6ZjbAHax zFT0=TcQdKj-KCoS*Y^e*RhE=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTpW)~HoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKmVM@&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^ZoF1e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS!c@e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aayN#do zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{Cp?; zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-IP z9)8Zx`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^WDbJ`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAJAnji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KC{1xHn{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIY0lV@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*aaN8itLJoPorsV|EjadvBQ*tdC)X{~n|SC46J*{gZ8=A(YT zyA}&)%?quA({Ykv#Pi1uom(9}s?jHP6-#xFl{(96oo_Ac*{CyZQdKj-KC zoS*aa?+ria=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`T4sye$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*aa?{EB^pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzkB6W0bAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{QQlLpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`S}NjpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e*U42pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T56%pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*OuKpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{QTkf^BhloO|xokpRA_KP8KuCa<#f)+i0BKS{(Lm-eX$pUB=a8T3hyN-mLkk zpYN{4!dde|>)>>pWEk=MaYN@;M~`asNnOQKonxiWvRda`%X&8IOk25N>C2 zmE6e9+{*1dmdA4^Pvprwm8bJeo{i^_=aA>n!~Yy|pZop5?+1QA@ILqZf!`1OexPnw zx2xOL?eh2X_wx7h_wx7h_wx7h_wx7h_wx7h_wx7h_wx6@+vM-%@8$31@A)}D=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luLL!_WCSKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJBWji2*#e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCd=h@n&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzQsd|R zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKc6*z z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS$EB{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;U(xtEKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzH-?|{bAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{QQ-TpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzyM&+fbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QP^u&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QUbGKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqSz>%!0ZIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*RQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv-L*wWC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKR;~z zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Hk z6n@Un`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^Pdkt=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqG=9#{`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6Zj^IvTIoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Hk8-C8u`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Zj^WO?T=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqwi zxm>wixm>wixm>wixm>wixm>wixm>wixm>wixm>y2-)VBWa=CK3a=CIcax!uQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=bsUN&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^UrGhoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-J;@N<67&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSze&gr- zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKi>{N z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC ze5dhqe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS%PrQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*aaS2TXk&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hwo9DbhTsjq2Pt?iT5blJ&bCRwglH*6b?vs;V9zRi0~YrV_3dQ5A}Ud@{|ANBLy zwOBZ7UT7Vhj*|={oQdKj-KCoS*Y^e$LPN`MWlL&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=dTSv=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KC{Pm5W^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luM~8$ajg{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`3Hud^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luNV!q53RKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luK^8b9af{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPN`NxEx^K*XA&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luLP!_WCSKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LMy4nNQF)Ymkt*7nJ2y6j{zlPp)O8@7$c*{#K4 z-{w80wccf1J*Kr~ujb8~kNWxUS}dG3FSHI$$4Q0}&mT8*ZgupiMxWGGEY&$y>MW~u zzO}4pqt3LIE4f-%d#$eXdR@tl+{~@q&SQBzck)D@%u{(f>J9aVdPBXT-cWC-Hy)09 z!~1+cv)s+S+|PqNm*?|BUd&5*Id92Z<99;86Z)M{-L7s|x2xOL?do=QySm+XHomj* zosI8od}rf38{gUZ&c=5(zO(V2&2Kl~+4#=JcQ(GWk(2SAjqhxHXCn{C&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj^G^>y=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`DZqM&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxzbO2ipYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QPS8IX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LO|8h*~t`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^J|Tt^K*XA z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luN38b9af z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`P1R& z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0mU z@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqM^Y?do^#?eALf(*J9zUd7*W1 zI!-c-c>cJdbE~7(8|n@9hI&K2q25q$T$XxcF-ui5tW?#oTJ^(P*0WKS!&a`ueeOPY zpS#cHa^-U6a=)?3<;vyC<;vyC<;vyC<;vyC<;vyC<;vyC<;vyC<;vyC<;vyC<;vyC z$;ipb$;ip@bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;@6z}=Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqgpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa*EN35&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;9}s@d&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz2Q_}q&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;A02+q&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz$2NY> z&-pn&=jZ&KpYwBm&d>QdKff$~p5v*nX;!W6lht(D$zmp1u2wf}8;!GDi^IOndrWJ+ z%eZ<>Ys+5En>8Qx^WC*rIBQ;L9h{Dn3?rUDZs^?VX!VAAL%pHiP;aO=)Ek$j-dN01 z)eI|DHLOQd zKj-KCoS*Y^e$LPNIX~y;pB8@3&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqSzXEc7!&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEVfZ;e=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luMO8$ajg{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`IYcQdKj-KCoS(lX{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;+l`;|bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QOHBKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqSzQ{m_QoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKmWS$bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6XZ)A%_*=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luNg_<4?}zNT5Vwog{mWhaZ7 zWVu@1ux&KXZY>V`Ht#X5^)BP;F|93oHE-5@)X#U=RNPBM&m{1z;phCEpYwBm&d>SzJBFY0bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QNt^&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QSEcKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqSztHaOvIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*VMZ=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv-ZR6+soS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKmXCj&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmVEVbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEVB_cfoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKmWPL&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmYacbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEOylSL zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv-bo@NWQ(x1pTH7b9>9Ui>OtM_9ZrC;&XSWuI zeVg}~)_Rw5^_bR{y_z>`KI-SYYq4QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luLr!_WCSKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0mO#?SdV zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*OjF z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8huy zHGa;|`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^Dk=roS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-IL;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqwb|pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzTN^*;=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`T3LK=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hxT+VFFJ&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{4)7@j;Fq+S+%xLR?}rCiTra8|sbARBtS1scMFmsv1_Sept(T zHmY)PpS#c9=k9a4T)AAi+^=tPxpKL3xpKL3xpKL3xpKL3xpKL3xpKL3xpKL3xpKL3 zxpKL3xpKL3GIBCQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzJ2igJ&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;uL?is=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T0TkIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LNd)A%_*=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zv#qe`}&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEK;!59 zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmXar z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze*PcB&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqW8(gXQL_y_qqGreeOP&%azNO%azNO%azNO%azNO%azNO%azNO z%azNO%azNO%azNO%l(Zemn)Ymmn)YmCnF~#CnG1r&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAJ9e!_WCSKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJA5ji2*#e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC{Er(y=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`9kC8{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0lx#?SdVKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6XZ5q{3k`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^M4qA&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS$DVKhN>h*EFlv_Q`6x z>|`;MELW==wvEQwt;J#A<~^ph-ep`prnP0S=FOUq`uXl!ESxnjv<^)j3w`EUR_CwXA2O&a{;)xms6yt*-NWUCE8y%&pwcV|hGx!uR<;-~YPC z_xV2G=lgu0@AG}W&-eL0-{<>$pYQX1zR&mhKHum2e4p?0eZJ4t_&(piT)xlG_cP1g z+{^tu$a8rQdKj-KCoS*aae;R(y&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;-`V&%Kj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;|1$iXpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;Kiv2^ zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqis z_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z|84j=Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIY0j^;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`T0jRe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zvm+*6b&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTqsGtqIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*SmD&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`4=>P z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKfe=x&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Zj^CufW=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqTs@|>Wv}MVnveSV z?piFIH7~RdPRB`x5ziktbZ&L@s79aERV>vxR_ZLPb-uN%$8*SY$aCnjJ%`-q`r>&ffM>&ffM>&ffM>&ffM>&ffM>&ffM>&ffM z>&ffM>&ffM>&ffM>&ffM>&ffM$?$W2&d>QdKj-KC{7;3S^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luMigrDQdKj-KC{2dxU=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Y;lq=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`KuZ~=jZ&K zpYwBm&d>QdKj-KC{Qt9a-@lb~|6Rb}$G6zN9(%pJUaxoe>E-)P2SIwA1c~&HAiaUo zI|$Mnh$Oz=@;u*`-j&`#KzacS5~l|Q>CFhz8;F(y!)NUKwJkY+z+>w*(rC`iJ#%J$ znK>Wd*ZZ2E^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN`9b67{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX{2T@N<67&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqi=QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luLb z!_WCSKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIY0kK7l^X6dYXp>=RNPBM&m{=A`ctFuQn`lRk+ zxvsHVS6Qp;t!E?Z4fTe4L%pHiP;aO=E?d1}KHtkc`?;S7d6-9eAuq;!Zaz1ko6pVX z=5zD8{Js3W{Js3W{Js3W{Js3W{Js3W{Js3W{Js3W{QZA!^7r!h^7r!h^7r!h^7r!h z@^Ji|pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTr0{co&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPTLF4EAoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmYHIpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^exAn9 z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ9$ z8b9af{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN z`9}CTKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzX5;7loS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKYwN8=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hv-BK(}6^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=U*Lu&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS$DkKhN>f*D|lx*2!+VY-gHDR;twv$42Aq)aI~niypJu?=r3)vpTX{>t?M- z{o>==ES)tkv<^`0?4|xw=z4wskd@u9t z=YAgKVIJj$c+NfNo^yFUc|Cc(uWj;r@_O=m@_O=m@_O=m@_O=m@_O=m@_O=m@_O=m z@_O=m@_O=m@_O=m@_O=m@_KSI{G6ZjbAHax`8hv-qwsTn&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPTYUAhpoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKmX^ApYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*Poj=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT8;zgybAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QTcEe$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa_Xt1d z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5U= zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`T4(Z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;9}<4f&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSz-);PypYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;m(S00y!5rqtF?8qn=ad# zW|Ea^b;GgII6JjD?AxNptoFN%tH-R4?AE$j>rub>xHd~?%?quA({Ykv#PjD3om-tf zs?jHP7t3{x)w;@BU2i?=4fTe4L%pHiP;aO=E?>Q2KHtkc`?;S7d6-9eA?9=Qx%u3D zZaz1ko6qI%QdKj-Iv9DdHv`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^S|HtIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPTTjS^a zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmVNY zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zn z;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lpy<{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;FNB}-bAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{QO$u=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hv-MdRoEoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKYu*@oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-IP6@Jdo`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6Zj^CufW=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqONYpm8)*6MoRL*7H)Ls#)VBV zJo~wy2YHxB@tk|kJ?HXz@_O=mU(@9EQdKmX3~bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZvmB!EcIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e*VuIKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqSzTZfQdKj-KCoS*Y^e$LPNIX~y;{QQT*&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QO@xe$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aacMm`3 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5U; zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`T5T^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*aa4-P-)=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`T4JhpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzW%TnLFMTcZYHgkDrptDw znPjC}-EeF)&Q5I(`?lyYtNkwH>M^S$yR~lCdQ?T>YnP?hHvW?B!Ra{3Fyi_1hVCs( z8P({Mx{Kwy#%f(QdKmVifbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6YEQsd|RoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmU`)&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmUK>=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hxT+{VxOIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e%^(j^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=luLp_&Go4=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxHyS_Z=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T2{DpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LOIX#AX?^K*XA&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=U?3}qo3z^>1&x+YwKh;UA8mL zBrDbGhGU~~c4~9jw?&Ux?RObhk69hrt#z~3qki#mZI;fO7g`6W<0Qj~=g%8Dw>o=N zqfhECmg^d;b(OWc-g-82t*&%EH*&M?_Ez2J?YfgYxtn|O9`YXY9=e?GAQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luMQ8b9af{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`FAyb&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX{25@N<67&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0lP z@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lq5I{G6ZjbAHax`8hx5=lqlHwQkmW)Gt1+&C*%(LhImkoMagB{CPv?R%efD^hw>t za$RG!uCi9wThB(W)s?R2MsC*K-m3e&U3YRPcXKb^L*7H)LznYCBVJo~wy2YDEO zSNOZa-xcO_^SSxld@h$Omn)Ymmn)Ymmn)Ymmn)Ymmn)Ymmn)Ymmn)Ymm;3RN%azNO z%azNO%azNO%axPi=lqQdKj-KCoS*Y^e$LPNIY0ly z@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIY0m7#?SdVKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIY0lu;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAG-Se$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aahvDb^oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKVNVBoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-H!G=9#{`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Zj^Lydv{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0l3@N<67&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqE`Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=luM#`gx9*zLt5lwoZ1_WjoVMvQn*XI5rw*r#6Ru zTlARKewT6enAMTpS~qJw>K7l^X6dYXp>=RNPBM&m{=A`ctFuQn`lRk+xvsHVS5a@M zH`E*I4fTe4L%nfX>kaeyUgp`){XB^I+QdKj-KC{0$pF z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj z^S25==jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KC{H+^5=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Zj^LGtD=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KC{M{Qr=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Zj^A8L^=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KC{DT`m=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqlHwQkmW)Gt1+&C*%(LhImkoMagB z{CPv?R%efD^hw>ta$RG!uHrr9J>)%fRo_FN^S#WopZj?b&$;K^b1ttZuP3i3uP3i3 zuP3i3uP3i3uP3i3uP3i3uP3i3uP3i3uP3i3ulI40*OS+i*OS+i*OS+i*OQas=lqQdKj-KCoS*Y^e$LPNIY0k{@N<67&-pn&=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0lS#?SdVKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIY0l4@N<67&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqN{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX|C;pYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJA) z@pFF8&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`T3WJpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e*Tq>pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lq@T{ zzxcQ|OJ~gst%K8Xl3~R2=M9}(ojt12Cv_Lgb&b`!%357-JsY`JSGt}XxmkC6tM2o5 z-N~JJ4|xxH4_)5(kmr0a^X%t-{9WPi3V&Ca&&}uNbMyJ@MJ`t^S1wmBS1wmBS1wmB zS1wmBS1wmBS1wmBS1wmBS1wmBS1wmBS1wmBS1wmhhM)6ue$LPNIX~y;{QTR(&-pn& z=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QNr_ zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz zTZW(WbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{QL*P&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Y^OZYiI=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-J~*7!L;=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6YEK=?U7=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-Hk)c83+=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lq zm%f&HwYE-n(`7r;OtMm~Za6j?XQwuYeOvUH)qa<8^_bO>-C8$mJ?a-9*JkOgd7*W1 zI!-c-c>cVhbE~sQHTtCPV!5udT31=C>#b)a*Xl~wb0atFZg16n-mW{j6YnAKA@8Bf z`yTR~?`59-+>gI2{9WPi3iG-7+Qd zKj-KCoS*Y^e$LN7zVUN@&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hxTjPP@Q&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LN7tMPMw&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=luMB_&Go4=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax*BU?P=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T4`f&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=U*0n&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KC{3{wi=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqbJ({49oemQv(}@2@o{aI&YBlm2dCpC!-(h48#=c-dsL%O z>MoY+8tM)8hI&K2q25q$s5dTiyQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hxT_QucoIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e*PBW=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hxT{_t~t&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hxT!N$+|IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*Vtk=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hxTiSToN&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hxTsm9OwIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*XU9=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxTQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT)yB{HIX~y;{G6ZjbAHax`8hwo%6^{X zrLSdPt*w*YblJ`{ldM#$8;*^}*{RK8-xfV)wcll2J!W-ex7N*CkNU;OwOKlAUT7Vh zj*|={oQdKj-Hk*Z4U<=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEdiXg%=jZ&KpYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-J4+4wm>=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAG-be$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aaS>xyY zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKY!5p zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN# z!q53RKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX}PK_&Go4=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHaxzr68te$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KC{Brwwj+efcd9}7qcGG1$(@e5bt!_9r8fT|AhkaZ0nALularKzhk=wOVM>YDS?qa#Fv07JItLv?2BiHIm*K;E`>uzt=ecrA+ zxs$uOm&fvWp2(AVDo^K`Je%k8d|sB9=XJvO`99z0`+T48^L@V0_xV2G=lgu0@AG}W z&-eL0-{<>$pYQX1-?#I9zR&l$8sF#pm)rOG`CjHRpPSFk=jLQdKj-KC zoS*Y^e$LOot?_ey&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LOoFZ`UJ^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=Wp5gIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN-pz(8l&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LN-Jp7!W^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=kL<^IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN-vhj0%&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LN-Dg2zD^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=O57cIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN-rSWrq&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHaxU)p}2xW zWVhDMT95k0$F*5HYhGv_oQ{(WBc4BR=-le;QH?&SyI8JktkzZ5>U!(h$hErC_1wtK zy4zcIpSSBy?&NOn<*_`TC-P*T%F}r!&*r&2pO@w3@gDLX@*a9=zlS{Mdzr`g2fjb> z{ek)1_XoZ|@cn_hUEQv3SGUXG%iqi2%iqi2%iqi2%iqi2%iqi2%in)=QdKj-KCoS*Y^e$LN-H~gHR z^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=O5en zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LN- zzwvW^&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPTH2j>O^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=bzE|IX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LOA!_WCSKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`T1(&=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hwo-}pH{=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=luLj!q53RKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS%PLQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqlHwQkmW)Gt1+&C*%(LhImkoMagB{CPv?R%efD^hw=?dPBXT-cWC-H`E*I zjmutdn9uiOJ~yA6&&}uNbMv|R{N<6qm%o?4m%o?4m%o?4m%o?4m%o?4m%o?4m%o?4 zm%o?4m%o?4m%o?4m%o?4mxtr${G6ZjbAHax`8hxT=J0cV&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT*2d5IIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e*R|R=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT-tcpN&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hxT{>IPwIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*TW(=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv73_s`R z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`8zj$ z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=kFVS&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPN`TIA1&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqV^&9YYu&8%s9$_so29enh1S98ILR>L`SXU(tQdKj-KC zoS*Y^e$LN-C;XhB^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=O5GfIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LN-ukmw!&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPTPxv`M=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;{G6Xp!_WCSKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJBC;phCEpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAEpL{XEA@U(38&TPM5evYlxrS*cbx z92dvvk(H&^kCBCmBXOf8Nl!)!CyOeNuO^ zT-R8wtE|=a*0Ygob*1aMk(+h5x9UD`*PYzS-Q3G#c|1?#$vl;(^Gu%2b9p{53*YDa ze4p?0eZJ54`99z0`+T48^L@V0_xV2G=lgu0@AG}W&-Z=b&iDB~-{)$4pYLCO-#4F| z&&}uNbMv|R+QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luLz8b9af{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`J0BH^K*XA&-pn&=jZ&KpYwBm&d>Qd zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luM8!q53RKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luNp8b9af{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPN`8$N4^K*XA&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luM~!q53RKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luM~8$ajg{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`TK;Q^K*XA z&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luK^!q53R zKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=luMa z8b9af{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHaxe>?n~pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHaxf4A{-e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS%P6_&Go4=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax|9AK~Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*OjF=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hvdZv32|^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=kvzT`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAEm+{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;Ule}M&-pn&=jZ&KpYwBm&d>QdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHaxuY#ZFcV{*Zadv8R*tbQGS?zZjSC3g8*{yZ6)}wy$ac!2)nipCJr{g5Ui0986I=4D|RHLgm z)Envz^@e&wy`kQ?3iZY`%T+V1R@Jap^}~8La;++d>$#DeRegBQJ?EZt&-wF_%azNO z%azNO%azNO%azNO%azNO%azNO%azNO%azNO%azNO%azNO%azNO%axOnlaZ5=li}z5 zoS*Y^e*TT&=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hv-dE@8&oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmX>&&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKmYFVbAHax`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6Y^S>xyYoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmXpw&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKmXD2bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Y^W8>%ioS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKR;~zoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-J~9e&Qw`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^Pdkt=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj^IvTIoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hwo3VxpBrLSdPt*w*YblJ`{ldM#$8;*^}*{RK8-xfV)wcll2J!W-ex7N*C zkNU;OwOKlAUT7Vhj*|={oTra8|n@9hI->F)Em<*SIw|mRl{1< z59`^;wW=Jh=SFT;_2D`9oO{ka=W@AnxpKL3xpKL3xpKL3xpKL3xpKL3xpKL3xpKL3 zxgQa^T)AAiT)AAiT)AAiTsavz895m_8Gg>s`8hx5=lqQdKj-Ja6@Jdo`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6Zj^N(u$oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-Ja)A%_*=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KFEBu_F^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=bzg6IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LOw;phCEpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS#n{Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSzUgPKdoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKi>*J=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCe7o^;e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS%PjQdKj-KCoS*Y^ ze$LPNIX~y;{G6ZjbAHax`8hx5=lqlHwQkmW)Gt1+&C*%(LhImkoMagB{CPv?R%efD^hw>t za$RG!uCi9wThB(W)s?R2MsC*K-m3e&U3YRPcXKa~QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{QQl>&-pn&=jZ&K zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*UJ7pYwBm z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T5(0 zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{QMmnKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSzk2QYI&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQd zKj-KCoS*aa{}g`C&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;ztH$OKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqM^S$yR~lCdekpIuFcX} z^Fr(3bev=u@%(v1=T>KrYV=9n#d2L^wXU*O*IUm>UMRz-`V(`jo;b$osHkw_??a4+4!B!hsW=1{LaSj zZ2Zo~?`-_e#_w$8Wc<#??`-_eMjnoz^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^ ze$LPTYxp@o=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZvcH`&#oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmSMJ=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hxTpW)~HoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKmU}*&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN`R9e7^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hx5=lp!q_&Go4=lqQdKj-KC zoS*Y^e$LPNIX~y;{G6ZjbAHaxzo7ARe$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS)weKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqSzYr@a@IX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^etxU*bAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6YEQRCSzRq^v2FMTcZYHgkDrptDwnPjC}-EeF) z&Q5I(`?lyYtNkwH>M^S$yR~lCdekpIuFcX}^Fr(3bev=u@%(v1=T>K{H`E*I4fTe4 zL%pHixGMF=G|N>ptX9>qR`tVrHgc^hhwHf!&$;K^bM85RHgdUgxpKL3xpKL3xpKL3 zxpKL3xpKL3xpKL3xpKL3xpKL3xpKL3xpKL3xpKL3GIBCQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=g&8O z&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=YKu?oS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KFGW?vM^K*XA&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=Wo*ZIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e$LOoyYX{=&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^ ze$LPNIX~y;{G6Zv+wgOK&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hxT(ZQdKj-KCoS*Y^e*U+^&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*WH#pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=lqtr`wwlmEnE7j_TW213)YIE4PMUPqScNtfYSsmG}b+gu^e(`Z_md=_N zS_h}&B*Td3&l@_oI$OP=-cWC-H`E*I4fV!VsW+xsu9{)Bs)n_yAJ(&xYgIX1&y9G_ zJ?EZt&$(Q#T&`TMT&`TMT&`TMT&`TMT&`TMT&`TMT&`TMT<(WOE>|vBE>|vBE>|vB zE>})QPDV~fPKKZJbAHax`8hx5=lqSze+fV5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`T0jQe$LPN zIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*aa-)j7v zpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z|0DdIpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHaxm%`8aIX~y;{G6ZjbAHax`8hx5=lqQdKj-KC zoS*Y^e*RCx&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAHax`8hx5=bzvBIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LNtgrDQdKj-KC{MF&-{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~ZO{G6ZjbAHax`8hx5=lqQd zKj-KCoS*Y^e$LPNIX~y;U)cCLKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqX(m~zRyQ0Qjk8mm!@ezg%xb^O zxO&X$$ZoBhwI210k887Z*1XU+v4)9`YW#I`1LR`CjJP&;2~e!#v6hc`>iZEAy&&&OPUz^JgNjC$A^3C$A^3C$A^3 zC$A^3C$A^3C$A^3C$A^3C$A^3C$A^3C$A^3C$A^3$Is>UQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAJ9@ zQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6Zj zbAJBU!q53RKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAJBDji2*#e$LPNIX~y;{G6Zv|90&*G>*Iu;P`*ry>_j+W}BwzHkYQ`G;OzO znr@%d=QJD=G>Aw%4hbR>jzfZoghMo*=NS)?C?Y{bf`|lB3343aJSB)o@QENI z!6$-<1QiJ)*8gq}s=ePkUmbiVOn2s+nc002IQdKmSgFpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KCoS*Y^e$LPNIX~y;{G6XZTHxpWoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5 z=lqQdKmS32pYwBm&d>QdKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=lqQdKmS?c=lqQdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hv-qQKAjIX~y;{G6ZjbAHax`8hx5 z=lqQdKj-KCoS*Y^e*TLBKj-KCoS*Y^e$LPNIX~y; z{G6ZjbAHax`8hx5=luMQM-fG|8O)aS-?Me@&Qr>E5w4KFN$uFyq7hXGJ zPu_1y#eBq`TyEsGoY#$o*M6N}cHG~5-PZlQdKj-KCoS*Y^e$LPNIX~y; z{QRi`Kj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqSz9}4`OpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqQd zKj-KCoS*aaza@Uo&-pn&=jZ&KpYwBm&d>QdKj-KCoS*Y^e$LPNIX~y;{G6ZjbAHax z`8hx5=lqQdKj-KC zoS*Y^e$LPNIX~y;g~ZSKIX~y;{G6ZjbAHax`8hx5=lqQdKj-KCoS*Y^e*SFY=lqQdKj-KCoS*Y^e$LPN zIX~y;{G6ZjbAHax`8hw|S>WgVoS*Y^e$LPNIX~y;{G6ZjbAHax`8hx5=lqg~2Ys-Q?JI}ZS=`qa_EKAW*vH#C!oJj2 z343{eXV@q9cZGd_dlcg?;YOP}nyQ4TpWC^U<)+cfKX; zTb(0eAARJlVPAOUv9PDEw}pMIYc%YOU3bG?I{ZY~#}B_F>`RBo!d~uvSJ)@I-yQbl z?)QYfa%4R0lSiHm`^u50!d|VuFYHs*iLkF$-yinc(GP@u`sfG4zIJpn?Dd|f!#>mV zOxV|ZJ{0!BV^d+DJ@(9BA1J{$It;~xwA{PB;6ee3v4*hf!1 z7xsk{&xbvod?D;(CuhUHc=E-tmujC2`*`j1VPC4vg}r?0i(#KQ^`)>cpZapxD}D1} zpX~ce*jM_#8usexuZ4Z;^g`HIPk%k^wf=8}eY*dfVPETC414{|x57Si=G$RkKl9zN z4-PDaeRkk`Vc!^dDeOaMUk>}++2ycro_!_kBlTCqK41S~*thB{VIMvBqp&ZW`*GOQ z`JaS+?EGrj7tjAR?4=7o5BvCqUxt0@!dlqN7k?G@iHpAu`|`!#guQZUJ?xX0ejE0c zOTQ0$_3|IXK6QB`?5mgm6!zMcKZkw#%3s31c4af{^})Y}eP;0QVP7BoN7x6iZiRjJ z>OaH2arIweAG-GMu+Lpf%?JMN|3%aN=l^Rv!aj1n820(=yTZP8y%hG*8+*dOaHA#c z>E_>$F%3QoGSU7YD&AC-&Wh3-(T}?vGKd1ymmEq^4eG$=e#$srREvM=D#h?{g%AF aySd*utZ}_!^Cw@wHKjXu@4o$==l%mHi9*x> diff --git a/tests/queries/0_stateless/test_dozlem/nullable_arrays.arrow b/tests/queries/0_stateless/test_dozlem/nullable_arrays.arrow deleted file mode 100644 index 2e497d358c6da6916baae0127b59487ae1f55ec9..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1322 zcmeHHyG{c^44lhdLKGG$#R*YRTnQ!6P!eh+P*Ok%5)q^bpd$qZ1tn!Zg97mdd;|rb z!9RqVJzJc@tDwcvu6Jzj*z3JbuixK4+yG{j;{@nJ(m_BAO|-e9BpES?+MF|+ft2W( z<1|`wNB_G_tD`Q@2oQ3&S@0zVyFFT`s`l&a0tZ5;3qielN8b6;Qi)ScAL}AL= zJ(H*>l7MCU)#(sfk!G>xY31hJAr8!6-NhR%%BSADEcxOKtyqa`oll~^NZhV7*Tu2> zx6SGMcMSVEjtnoFJ~!>kvGd)S!};zEyI)XuUq7d8ewPga%6ERZyRM=??{^OIYBVj- z>+(GFado?VI=2tu!R2^ZTu;i;#hG9f)iR&QT*#l+ceU2{A+0_HekFPZfk^L9yLAt` b8}X>dD=T#Bnzp+9zuCXNS-sU>{EvJGKXY;* From 9013892a965d186e229bb1e64c28cd125d5cdd8f Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 19 Aug 2021 20:56:16 +0300 Subject: [PATCH 217/220] make the sql-standard window functions case insensitive --- src/Processors/Transforms/WindowTransform.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Processors/Transforms/WindowTransform.cpp b/src/Processors/Transforms/WindowTransform.cpp index 1b8406682ea..132bdb7b327 100644 --- a/src/Processors/Transforms/WindowTransform.cpp +++ b/src/Processors/Transforms/WindowTransform.cpp @@ -1764,21 +1764,21 @@ void registerWindowFunctions(AggregateFunctionFactory & factory) { return std::make_shared(name, argument_types, parameters); - }, properties}); + }, properties}, AggregateFunctionFactory::CaseInsensitive); factory.registerFunction("dense_rank", {[](const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { return std::make_shared(name, argument_types, parameters); - }, properties}); + }, properties}, AggregateFunctionFactory::CaseInsensitive); factory.registerFunction("row_number", {[](const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { return std::make_shared(name, argument_types, parameters); - }, properties}); + }, properties}, AggregateFunctionFactory::CaseInsensitive); factory.registerFunction("lagInFrame", {[](const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) @@ -1799,7 +1799,7 @@ void registerWindowFunctions(AggregateFunctionFactory & factory) { return std::make_shared( name, argument_types, parameters); - }, properties}); + }, properties}, AggregateFunctionFactory::CaseInsensitive); } } From 941eba908c406bbfadc90ae8ed01987603512f57 Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Thu, 19 Aug 2021 21:38:06 +0300 Subject: [PATCH 218/220] Bump librdkafka (to fix metadata cache destroying) This should fix CI under TSan [1]. [1]: https://clickhouse-test-reports.s3.yandex.net/0/9292869c4f92664a28b8c9ddef1e62ddfd13b285/integration_tests_(thread).html Refs: edenhill/librdkafka#3279 --- contrib/librdkafka | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/librdkafka b/contrib/librdkafka index 43491d33ca2..b8554f16820 160000 --- a/contrib/librdkafka +++ b/contrib/librdkafka @@ -1 +1 @@ -Subproject commit 43491d33ca2826531d1e3cae70d4bf1e5249e3c9 +Subproject commit b8554f1682062c85ba519eb54ef2f90e02b812cb From a7d405759cabc85f7a3a5ada99943102eb32274c Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 19 Aug 2021 21:43:43 +0300 Subject: [PATCH 219/220] fix decimal formatting settings in perf test --- docker/test/performance-comparison/compare.sh | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/docker/test/performance-comparison/compare.sh b/docker/test/performance-comparison/compare.sh index e5c9f349ce3..c97e8a6ed2b 100755 --- a/docker/test/performance-comparison/compare.sh +++ b/docker/test/performance-comparison/compare.sh @@ -628,9 +628,6 @@ cat analyze/errors.log >> report/errors.log ||: cat profile-errors.log >> report/errors.log ||: clickhouse-local --query " --- We use decimals specifically to get fixed-point, fixed-width formatting. -set output_format_decimal_trailing_zeros = 1; - create view query_display_names as select * from file('analyze/query-display-names.tsv', TSV, 'test text, query_index int, query_display_name text') @@ -644,6 +641,7 @@ create view partial_query_times as select * from -- Report for partial queries that we could only run on the new server (e.g. -- queries with new functions added in the tested PR). create table partial_queries_report engine File(TSV, 'report/partial-queries-report.tsv') + settings output_format_decimal_trailing_zeros = 1 as select toDecimal64(time_median, 3) time, toDecimal64(time_stddev / time_median, 3) relative_time_stddev, test, query_index, query_display_name @@ -716,8 +714,9 @@ create table queries engine File(TSVWithNamesAndTypes, 'report/queries.tsv') order by test, query_index, metric_name ; -create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv') as - with +create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv') + settings output_format_decimal_trailing_zeros = 1 + as with -- server_time is sometimes reported as zero (if it's less than 1 ms), -- so we have to work around this to not get an error about conversion -- of NaN to decimal. @@ -733,8 +732,9 @@ create table changed_perf_report engine File(TSV, 'report/changed-perf.tsv') as changed_fail, test, query_index, query_display_name from queries where changed_show order by abs(diff) desc; -create table unstable_queries_report engine File(TSV, 'report/unstable-queries.tsv') as - select +create table unstable_queries_report engine File(TSV, 'report/unstable-queries.tsv') + settings output_format_decimal_trailing_zeros = 1 + as select toDecimal64(left, 3), toDecimal64(right, 3), toDecimal64(diff, 3), toDecimal64(stat_threshold, 3), unstable_fail, test, query_index, query_display_name from queries where unstable_show order by stat_threshold desc; @@ -764,8 +764,9 @@ create view total_speedup as from test_speedup ; -create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes.tsv') as - with +create table test_perf_changes_report engine File(TSV, 'report/test-perf-changes.tsv') + settings output_format_decimal_trailing_zeros = 1 + as with (times_speedup >= 1 ? '-' || toString(toDecimal64(times_speedup, 3)) || 'x' : '+' || toString(toDecimal64(1 / times_speedup, 3)) || 'x') @@ -791,8 +792,9 @@ create view total_client_time_per_query as select * from file('analyze/client-times.tsv', TSV, 'test text, query_index int, client float, server float'); -create table slow_on_client_report engine File(TSV, 'report/slow-on-client.tsv') as - select client, server, toDecimal64(client/server, 3) p, +create table slow_on_client_report engine File(TSV, 'report/slow-on-client.tsv') + settings output_format_decimal_trailing_zeros = 1 + as select client, server, toDecimal64(client/server, 3) p, test, query_display_name from total_client_time_per_query left join query_display_names using (test, query_index) where p > toDecimal64(1.02, 3) order by p desc; @@ -877,8 +879,9 @@ create view test_times_view_total as from test_times_view ; -create table test_times_report engine File(TSV, 'report/test-times.tsv') as - select +create table test_times_report engine File(TSV, 'report/test-times.tsv') + settings output_format_decimal_trailing_zeros = 1 + as select test, toDecimal64(real, 3), toDecimal64(total_client_time, 3), @@ -896,8 +899,9 @@ create table test_times_report engine File(TSV, 'report/test-times.tsv') as ; -- report for all queries page, only main metric -create table all_tests_report engine File(TSV, 'report/all-queries.tsv') as - with +create table all_tests_report engine File(TSV, 'report/all-queries.tsv') + settings output_format_decimal_trailing_zeros = 1 + as with -- server_time is sometimes reported as zero (if it's less than 1 ms), -- so we have to work around this to not get an error about conversion -- of NaN to decimal. @@ -978,9 +982,6 @@ for version in {right,left} do rm -rf data clickhouse-local --query " --- We use decimals specifically to get fixed-point, fixed-width formatting. -set output_format_decimal_trailing_zeros = 1; - create view query_profiles as with 0 as left, 1 as right select * from file('analyze/query-profiles.tsv', TSV, @@ -1063,9 +1064,10 @@ create table unstable_run_traces engine File(TSVWithNamesAndTypes, ; create table metric_devation engine File(TSVWithNamesAndTypes, - 'report/metric-deviation.$version.tsv') as + 'report/metric-deviation.$version.tsv') + settings output_format_decimal_trailing_zeros = 1 -- first goes the key used to split the file with grep - select test, query_index, query_display_name, + as select test, query_index, query_display_name, toDecimal64(d, 3) d, q, metric from ( select @@ -1176,9 +1178,6 @@ rm -rf metrics ||: mkdir metrics clickhouse-local --query " --- We use decimals specifically to get fixed-point, fixed-width formatting. -set output_format_decimal_trailing_zeros = 1; - create view right_async_metric_log as select * from file('right-async-metric-log.tsv', TSVWithNamesAndTypes, '$(cat right-async-metric-log.tsv.columns)') @@ -1196,8 +1195,9 @@ create table metrics engine File(TSV, 'metrics/metrics.tsv') as ; -- Show metrics that have changed -create table changes engine File(TSV, 'metrics/changes.tsv') as - select metric, left, right, +create table changes engine File(TSV, 'metrics/changes.tsv') + settings output_format_decimal_trailing_zeros = 1 + as select metric, left, right, toDecimal64(diff, 3), toDecimal64(times_diff, 3) from ( select metric, median(left) as left, median(right) as right, From 64bfe21a1be96afe31df48c53df4ca408113d776 Mon Sep 17 00:00:00 2001 From: tavplubix Date: Fri, 20 Aug 2021 00:25:14 +0300 Subject: [PATCH 220/220] Fix test 00443_preferred_block_size_bytes.sh (#27846) * Update 00443_preferred_block_size_bytes.sh * Update clickhouse-test * Update clickhouse-test * Update database_replicated.xml --- tests/clickhouse-test | 4 +++- tests/config/config.d/database_replicated.xml | 6 +++--- .../queries/0_stateless/00443_preferred_block_size_bytes.sh | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/clickhouse-test b/tests/clickhouse-test index c627810a550..0d833e5fbe6 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -44,15 +44,17 @@ DISTRIBUTED_DDL_TIMEOUT_MSG = "is executing longer than distributed_ddl_task_tim MESSAGES_TO_RETRY = [ "DB::Exception: ZooKeeper session has been expired", - "DB::Exception: Connection loss", "Coordination::Exception: Session expired", "Coordination::Exception: Connection loss", "Coordination::Exception: Operation timeout", + "DB::Exception: Session expired", + "DB::Exception: Connection loss", "DB::Exception: Operation timeout", "Operation timed out", "ConnectionPoolWithFailover: Connection failed at try", "DB::Exception: New table appeared in database being dropped or detached. Try again", "is already started to be removing by another replica right now", + "DB::Exception: Cannot enqueue query", "Shutdown is called for table", # It happens in SYSTEM SYNC REPLICA query if session with ZooKeeper is being reinitialized. DISTRIBUTED_DDL_TIMEOUT_MSG # FIXME ] diff --git a/tests/config/config.d/database_replicated.xml b/tests/config/config.d/database_replicated.xml index 9a3b4d68ea6..e51d90dd4d4 100644 --- a/tests/config/config.d/database_replicated.xml +++ b/tests/config/config.d/database_replicated.xml @@ -22,9 +22,9 @@ 10000 30000 1000 - 2000 - 4000 - trace + 4000 + 5000 + information false 1000000000000000 diff --git a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh index 724630057d9..399a4677a44 100755 --- a/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh +++ b/tests/queries/0_stateless/00443_preferred_block_size_bytes.sh @@ -42,10 +42,10 @@ popd > /dev/null #SCRIPTDIR=`dirname "$SCRIPTPATH"` SCRIPTDIR=$SCRIPTPATH -cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=10 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout 2>&1 +cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=10 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout cmp "$SCRIPTDIR"/00282_merging.reference "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout && echo PASSED || echo FAILED -cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=20 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout 2>&1 +cat "$SCRIPTDIR"/00282_merging.sql | $CLICKHOUSE_CLIENT --preferred_block_size_bytes=20 -n > "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout cmp "$SCRIPTDIR"/00282_merging.reference "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout && echo PASSED || echo FAILED rm "${CLICKHOUSE_TMP}"/preferred_block_size_bytes.stdout